mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-05-07 05:20:59 +08:00
refactor: 清理 parser worker rag merger 的历史类型问题
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
|
||||
import { getActiveConfig, buildPiModel } from '../llm'
|
||||
import { getAllTools, createActivateSkillTool } from '../tools'
|
||||
import type { ToolContext, OwnerInfo } from '../tools/types'
|
||||
import type { ToolContext } from '../tools/types'
|
||||
import { getHistoryForAgent } from '../conversations'
|
||||
import { aiLogger, isDebugMode } from '../logger'
|
||||
import { t as i18nT } from '../../i18n'
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
type Usage as PiUsage,
|
||||
} from '@mariozechner/pi-ai'
|
||||
|
||||
import type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, SkillContext } from './types'
|
||||
import type { AgentConfig, AgentStreamChunk, AgentResult, SkillContext } from './types'
|
||||
import type { AssistantConfig } from '../assistant/types'
|
||||
import { buildSystemPrompt } from './prompt-builder'
|
||||
import { extractThinkingContent, stripToolCallTags } from './content-parser'
|
||||
@@ -25,7 +25,7 @@ import { AgentEventHandler } from './event-handler'
|
||||
type SimpleHistoryMessage = { role: 'user' | 'assistant'; content: string }
|
||||
|
||||
// Re-export types for external consumers
|
||||
export type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, AgentRuntimeStatus } from './types'
|
||||
export type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, AgentRuntimeStatus, SkillContext } from './types'
|
||||
|
||||
/**
|
||||
* Agent 执行器类
|
||||
@@ -120,7 +120,8 @@ export class Agent {
|
||||
if (newMessages.length > 0) {
|
||||
const parts: string[] = []
|
||||
for (const m of newMessages) {
|
||||
const msg = m as Record<string, unknown>
|
||||
// pi-ai 的 Message 联合类型较严格,这里仅用于调试日志读取动态字段。
|
||||
const msg = m as unknown as Record<string, unknown>
|
||||
parts.push(`--- ${msg.role} ---`)
|
||||
const content = msg.content as
|
||||
| Array<{ type: string; text?: string; name?: string; arguments?: unknown }>
|
||||
|
||||
@@ -5,8 +5,9 @@
|
||||
* 通过 pluginQuery 执行参数化 SQL 并格式化结果。
|
||||
*/
|
||||
|
||||
import { Type, type TObject, type TProperties } from '@mariozechner/pi-ai'
|
||||
import type { AgentTool } from '@mariozechner/pi-agent-core'
|
||||
import { Type } from '@mariozechner/pi-ai'
|
||||
import type { AgentTool, AgentToolResult } from '@mariozechner/pi-agent-core'
|
||||
import type { TSchema } from '@sinclair/typebox'
|
||||
import type { ToolContext } from '../tools/types'
|
||||
import type { CustomSqlToolDef, JsonSchemaObject } from './types'
|
||||
import * as workerManager from '../../worker/workerManager'
|
||||
@@ -17,8 +18,8 @@ import { t as i18nT } from '../../i18n'
|
||||
*
|
||||
* 仅覆盖 SQL 工具参数定义的常见类型(string / number / integer / boolean)。
|
||||
*/
|
||||
export function jsonSchemaToTypeBox(schema: JsonSchemaObject): TObject<TProperties> {
|
||||
const props: TProperties = {}
|
||||
export function jsonSchemaToTypeBox(schema: JsonSchemaObject) {
|
||||
const props: Record<string, TSchema> = {}
|
||||
|
||||
for (const [key, prop] of Object.entries(schema.properties)) {
|
||||
const isRequired = schema.required?.includes(key) ?? false
|
||||
@@ -77,13 +78,21 @@ export function createSqlTool(def: CustomSqlToolDef, context: ToolContext): Agen
|
||||
label: def.name,
|
||||
description: def.description,
|
||||
parameters: schema,
|
||||
execute: async (_toolCallId: string, params: Record<string, unknown>) => {
|
||||
execute: async (
|
||||
_toolCallId: string,
|
||||
params: Record<string, unknown>,
|
||||
_signal?: AbortSignal,
|
||||
_onUpdate?: unknown
|
||||
): Promise<AgentToolResult<{ rows: Record<string, unknown>[]; rowCount: number }>> => {
|
||||
const rows = await workerManager.pluginQuery(context.sessionId, def.execution.query, params)
|
||||
|
||||
const fallback = resolveTemplate(def.name, 'fallback', def.execution.fallback)
|
||||
|
||||
if (!rows || rows.length === 0) {
|
||||
return { content: [{ type: 'text' as const, text: fallback }] }
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: fallback }],
|
||||
details: { rows: [], rowCount: 0 },
|
||||
}
|
||||
}
|
||||
|
||||
const rowTemplate = resolveTemplate(def.name, 'rowTemplate', def.execution.rowTemplate)
|
||||
@@ -102,7 +111,10 @@ export function createSqlTool(def: CustomSqlToolDef, context: ToolContext): Agen
|
||||
lines.push(formatRow(rowTemplate, row as Record<string, unknown>))
|
||||
}
|
||||
|
||||
return { content: [{ type: 'text' as const, text: lines.join('\n') }] }
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: lines.join('\n') }],
|
||||
details: { rows, rowCount: rows.length },
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { getLogsDir, ensureDir } from '../paths'
|
||||
import { getLogsDir } from '../paths'
|
||||
|
||||
let debugMode = false
|
||||
|
||||
|
||||
@@ -308,7 +308,7 @@ export function getSessionChunks(dbPath: string, options: ChunkingOptions = {}):
|
||||
|
||||
return chunks
|
||||
} catch (error) {
|
||||
logger.error('[Chunking] Failed to get session chunks:', error)
|
||||
logger.error('Chunking', 'Failed to get session chunks', error)
|
||||
return []
|
||||
} finally {
|
||||
if (db) {
|
||||
@@ -372,7 +372,7 @@ export function getSessionChunk(dbPath: string, sessionId: number): Chunk | null
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('[Chunking] Failed to get single session chunk:', error)
|
||||
logger.error('Chunking', 'Failed to get single session chunk', error)
|
||||
return null
|
||||
} finally {
|
||||
if (db) {
|
||||
|
||||
@@ -62,7 +62,7 @@ async function rewriteQuery(query: string, abortSignal?: AbortSignal): Promise<s
|
||||
|
||||
return rewritten || query
|
||||
} catch (error) {
|
||||
logger.warn('[Semantic Pipeline] Query rewrite failed, using original query:', error)
|
||||
logger.warn('Semantic Pipeline', 'Query rewrite failed, using original query', error)
|
||||
return query
|
||||
}
|
||||
}
|
||||
@@ -197,7 +197,8 @@ export async function executeSemanticPipeline(options: SemanticPipelineOptions):
|
||||
chunkVectors.set(chunk.id, vector)
|
||||
|
||||
if (vectorStore) {
|
||||
await vectorStore.add(chunk.id, vector, chunk.metadata as Record<string, unknown>)
|
||||
// ChunkMetadata 是结构化对象,这里仅在向量存储层按通用元数据字典处理。
|
||||
await vectorStore.add(chunk.id, vector, chunk.metadata as unknown as Record<string, unknown>)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ export async function getVectorStore(): Promise<IVectorStore | null> {
|
||||
activeStore = await createVectorStore(config.vectorStore)
|
||||
return activeStore
|
||||
} catch (error) {
|
||||
logger.error('[Store Manager] Failed to create store:', error)
|
||||
logger.error('Store Manager', 'Failed to create store', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -44,13 +44,13 @@ async function createVectorStore(config: VectorStoreConfig): Promise<IVectorStor
|
||||
switch (config.type) {
|
||||
case 'memory': {
|
||||
const capacity = config.memoryCacheSize || 10000
|
||||
logger.info(`[Store Manager] Using memory store, capacity: ${capacity}`)
|
||||
logger.info('Store Manager', `Using memory store, capacity: ${capacity}`)
|
||||
return new MemoryVectorStore(capacity)
|
||||
}
|
||||
|
||||
case 'sqlite': {
|
||||
const dbPath = config.dbPath || path.join(getVectorStoreDir(), 'embeddings.db')
|
||||
logger.info(`[Store Manager] Using SQLite store: ${dbPath}`)
|
||||
logger.info('Store Manager', `Using SQLite store: ${dbPath}`)
|
||||
return new SQLiteVectorStore(dbPath)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ export async function resetVectorStore(): Promise<void> {
|
||||
if (activeStore) {
|
||||
await activeStore.close()
|
||||
activeStore = null
|
||||
logger.info('[Store Manager] Store reset')
|
||||
logger.info('Store Manager', 'Store reset')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -224,7 +224,7 @@ export class MemoryVectorStore implements IVectorStore {
|
||||
this.cache.clear()
|
||||
this.head = null
|
||||
this.tail = null
|
||||
logger.info('[Memory Store] All vectors cleared')
|
||||
logger.info('Memory Store', 'All vectors cleared')
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -258,6 +258,6 @@ export class MemoryVectorStore implements IVectorStore {
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
// 内存存储无需关闭操作
|
||||
logger.info('[Memory Store] Closed')
|
||||
logger.info('Memory Store', 'Closed')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ export class SQLiteVectorStore implements IVectorStore {
|
||||
// 索引可能已存在
|
||||
}
|
||||
|
||||
logger.info(`[SQLite Store] Initialized: ${this.dbPath}`)
|
||||
logger.info('SQLite Store', `Initialized: ${this.dbPath}`)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,8 +107,9 @@ export class SQLiteVectorStore implements IVectorStore {
|
||||
VALUES (?, ?, ?, ?)
|
||||
`)
|
||||
|
||||
const insertMany = this.db.transaction((items: typeof items) => {
|
||||
for (const item of items) {
|
||||
type VectorBatchItem = { id: string; vector: number[]; metadata?: Record<string, unknown> }
|
||||
const insertMany = this.db.transaction((batchItems: VectorBatchItem[]) => {
|
||||
for (const item of batchItems) {
|
||||
const buffer = vectorToBuffer(item.vector)
|
||||
insert.run(item.id, buffer, item.vector.length, item.metadata ? JSON.stringify(item.metadata) : null)
|
||||
}
|
||||
@@ -180,7 +181,7 @@ export class SQLiteVectorStore implements IVectorStore {
|
||||
*/
|
||||
async clear(): Promise<void> {
|
||||
this.db.exec('DELETE FROM vectors')
|
||||
logger.info('[SQLite Store] All vectors cleared')
|
||||
logger.info('SQLite Store', 'All vectors cleared')
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -214,6 +215,6 @@ export class SQLiteVectorStore implements IVectorStore {
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
this.db.close()
|
||||
logger.info('[SQLite Store] Closed')
|
||||
logger.info('SQLite Store', 'Closed')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,7 +142,7 @@ function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): Agen
|
||||
const originalExecute = tool.execute
|
||||
return {
|
||||
...tool,
|
||||
execute: async (toolCallId: string, params: any) => {
|
||||
execute: async (toolCallId: string, params: any, _signal?: AbortSignal, _onUpdate?: unknown) => {
|
||||
const result = await originalExecute(toolCallId, params)
|
||||
|
||||
const details = result.details as Record<string, unknown> | undefined
|
||||
@@ -160,8 +160,8 @@ function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): Agen
|
||||
|
||||
const formatted = processed.map((m) => formatMessageCompact(m, context.locale))
|
||||
|
||||
const finalDetails = { ...details, messages: formatted, returned: processed.length }
|
||||
delete finalDetails.rawMessages
|
||||
const { rawMessages: _rawMessages, ...restDetails } = details
|
||||
const finalDetails = { ...restDetails, messages: formatted, returned: processed.length }
|
||||
|
||||
let textContent = formatToolResultAsText(finalDetails)
|
||||
if (nameMapLine) {
|
||||
@@ -242,6 +242,7 @@ export function createActivateSkillTool(
|
||||
|
||||
return {
|
||||
name: 'activate_skill',
|
||||
label: 'activate_skill',
|
||||
description: isZh
|
||||
? '激活一个分析技能,获取该技能的详细执行指导'
|
||||
: 'Activate an analysis skill and get its detailed execution instructions',
|
||||
@@ -255,11 +256,12 @@ export function createActivateSkillTool(
|
||||
},
|
||||
required: ['skill_id'],
|
||||
},
|
||||
execute: async (_toolCallId: string, params: { skill_id: string }) => {
|
||||
execute: async (_toolCallId: string, params: { skill_id: string }, _signal?: AbortSignal, _onUpdate?: unknown) => {
|
||||
const skill: SkillDef | null = getSkillConfig(params.skill_id)
|
||||
if (!skill) {
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: isZh ? '技能不存在' : 'Skill not found' }],
|
||||
details: { skillId: params.skill_id, found: false },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,7 +269,10 @@ export function createActivateSkillTool(
|
||||
const scopeMsg = isZh
|
||||
? `该技能仅适用于${skill.chatScope === 'group' ? '群聊' : '私聊'}场景`
|
||||
: `This skill is only applicable to ${skill.chatScope === 'group' ? 'group chat' : 'private chat'} scenarios`
|
||||
return { content: [{ type: 'text' as const, text: scopeMsg }] }
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: scopeMsg }],
|
||||
details: { skillId: params.skill_id, found: true, applicable: false },
|
||||
}
|
||||
}
|
||||
|
||||
if (skill.tools.length > 0 && allowedTools && allowedTools.length > 0) {
|
||||
@@ -276,7 +281,10 @@ export function createActivateSkillTool(
|
||||
const msg = isZh
|
||||
? `当前助手缺少该技能所需的工具:${missing.join(', ')}`
|
||||
: `Current assistant lacks tools required by this skill: ${missing.join(', ')}`
|
||||
return { content: [{ type: 'text' as const, text: msg }] }
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: msg }],
|
||||
details: { skillId: params.skill_id, found: true, applicable: false, missingTools: missing },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,6 +294,7 @@ export function createActivateSkillTool(
|
||||
|
||||
return {
|
||||
content: [{ type: 'text' as const, text: `${skill.prompt}${actionPrompt}` }],
|
||||
details: { skillId: params.skill_id, found: true, applicable: true },
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -26,7 +26,9 @@ export const i18nTexts = {
|
||||
},
|
||||
}
|
||||
|
||||
export function t(key: keyof typeof i18nTexts, locale?: string): string | string[] {
|
||||
type TextEntryKey = Exclude<keyof typeof i18nTexts, 'dailySummary'>
|
||||
|
||||
export function t(key: TextEntryKey, locale?: string): string | string[] {
|
||||
const text = i18nTexts[key]
|
||||
if (typeof text === 'object' && 'zh' in text && 'en' in text) {
|
||||
return isChineseLocale(locale) ? text.zh : text.en
|
||||
|
||||
@@ -328,6 +328,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
originators: [],
|
||||
initiators: [],
|
||||
breakers: [],
|
||||
fastestRepeaters: [],
|
||||
originatorRates: [],
|
||||
initiatorRates: [],
|
||||
breakerRates: [],
|
||||
@@ -389,16 +390,16 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
|
||||
const contentStats = new Map<
|
||||
string,
|
||||
{ count: number; maxChainLength: number; originatorId: number; lastTs: number }
|
||||
{ count: number; maxChainLength: number; originatorId: number; lastTs: number; firstMessageId: number }
|
||||
>()
|
||||
|
||||
let currentContent: string | null = null
|
||||
let repeatChain: Array<{ senderId: number; content: string; ts: number }> = []
|
||||
let repeatChain: Array<{ id: number; senderId: number; content: string; ts: number }> = []
|
||||
let totalRepeatChains = 0
|
||||
let totalChainLength = 0
|
||||
|
||||
const processRepeatChain = (
|
||||
chain: Array<{ senderId: number; content: string; ts: number }>,
|
||||
chain: Array<{ id: number; senderId: number; content: string; ts: number }>,
|
||||
breakerId?: number
|
||||
) => {
|
||||
if (chain.length < 3) return
|
||||
@@ -421,6 +422,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
|
||||
const content = chain[0].content
|
||||
const chainTs = chain[0].ts
|
||||
const firstMessageId = chain[0].id
|
||||
const existing = contentStats.get(content)
|
||||
if (existing) {
|
||||
existing.count++
|
||||
@@ -428,9 +430,16 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
if (chainLength > existing.maxChainLength) {
|
||||
existing.maxChainLength = chainLength
|
||||
existing.originatorId = originatorId
|
||||
existing.firstMessageId = firstMessageId
|
||||
}
|
||||
} else {
|
||||
contentStats.set(content, { count: 1, maxChainLength: chainLength, originatorId, lastTs: chainTs })
|
||||
contentStats.set(content, {
|
||||
count: 1,
|
||||
maxChainLength: chainLength,
|
||||
originatorId,
|
||||
lastTs: chainTs,
|
||||
firstMessageId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,13 +455,13 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
if (content === currentContent) {
|
||||
const lastSender = repeatChain[repeatChain.length - 1]?.senderId
|
||||
if (lastSender !== msg.senderId) {
|
||||
repeatChain.push({ senderId: msg.senderId, content, ts: msg.ts })
|
||||
repeatChain.push({ id: msg.id, senderId: msg.senderId, content, ts: msg.ts })
|
||||
}
|
||||
} else {
|
||||
processRepeatChain(repeatChain, msg.senderId)
|
||||
|
||||
currentContent = content
|
||||
repeatChain = [{ senderId: msg.senderId, content, ts: msg.ts }]
|
||||
repeatChain = [{ id: msg.id, senderId: msg.senderId, content, ts: msg.ts }]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,6 +518,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
maxChainLength: stats.maxChainLength,
|
||||
originatorName: originatorInfo?.name || '未知',
|
||||
lastTs: stats.lastTs,
|
||||
firstMessageId: stats.firstMessageId,
|
||||
})
|
||||
}
|
||||
hotContents.sort((a, b) => b.maxChainLength - a.maxChainLength)
|
||||
@@ -518,6 +528,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
|
||||
originators: buildRankList(originatorCount, totalRepeatChains),
|
||||
initiators: buildRankList(initiatorCount, totalRepeatChains),
|
||||
breakers: buildRankList(breakerCount, totalRepeatChains),
|
||||
fastestRepeaters: [],
|
||||
originatorRates: buildRateList(originatorCount),
|
||||
initiatorRates: buildRateList(initiatorCount),
|
||||
breakerRates: buildRateList(breakerCount),
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import type { DbMeta, ParseResult, AnalysisSession } from '../../../src/types/base'
|
||||
import type { ParseResult } from '../../../src/types/base'
|
||||
import { migrateDatabase, needsMigration, CURRENT_SCHEMA_VERSION } from './migrations'
|
||||
import { getDatabaseDir, ensureDir } from '../paths'
|
||||
|
||||
@@ -162,7 +162,6 @@ export function openDatabaseWithMigration(sessionId: string, forceRepair = false
|
||||
*/
|
||||
export function importData(parseResult: ParseResult): string {
|
||||
const sessionId = generateSessionId()
|
||||
const dbPath = getDbPath(sessionId)
|
||||
const db = createDatabase(sessionId)
|
||||
|
||||
try {
|
||||
@@ -293,17 +292,12 @@ export function importData(parseResult: ParseResult): string {
|
||||
})
|
||||
|
||||
importTransaction()
|
||||
|
||||
const fileExists = fs.existsSync(dbPath)
|
||||
|
||||
return sessionId
|
||||
} catch (error) {
|
||||
console.error('[Database] Error in importData:', error)
|
||||
throw error
|
||||
} finally {
|
||||
db.close()
|
||||
|
||||
const fileExists = fs.existsSync(dbPath)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -237,7 +237,7 @@ class MainProcess {
|
||||
})
|
||||
|
||||
// 监听渲染进程崩溃
|
||||
app.on('render-process-gone', (e, w, d) => {
|
||||
app.on('render-process-gone', (_event, w, d) => {
|
||||
if (d.reason == 'crashed') {
|
||||
w.reload()
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import * as worker from '../worker/workerManager'
|
||||
import * as parser from '../parser'
|
||||
import { detectFormat, diagnoseFormat, scanMultiChatFile, type ParseProgress } from '../parser'
|
||||
import type { IpcContext } from './types'
|
||||
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos, type MigrationInfo } from '../database/migrations'
|
||||
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos } from '../database/migrations'
|
||||
import { exportSessionToTempFile, cleanupTempExportFiles } from '../merger'
|
||||
import { t } from '../i18n'
|
||||
|
||||
|
||||
+143
-17
@@ -9,21 +9,14 @@ import { parseFileSync, detectFormat } from '../parser'
|
||||
import { importData } from '../database/core'
|
||||
import { TempDbReader } from './tempCache'
|
||||
import { getDownloadsDir } from '../paths'
|
||||
import type { ParseResult, ParsedMessage, ChatPlatform, ChatType, ParsedMember } from '../../../src/types/base'
|
||||
import type {
|
||||
ParseResult,
|
||||
ParsedMessage,
|
||||
ChatLabFormat,
|
||||
ChatLabMember,
|
||||
ChatLabMessage,
|
||||
FileParseInfo,
|
||||
MergeConflict,
|
||||
ChatPlatform,
|
||||
ChatType,
|
||||
ParsedMember,
|
||||
} from '../../../src/types/base'
|
||||
import type {
|
||||
ConflictCheckResult,
|
||||
ConflictResolution,
|
||||
MergeParams,
|
||||
MergeResult,
|
||||
MergeSource,
|
||||
@@ -83,6 +76,10 @@ function getMessageKey(msg: ParsedMessage): string {
|
||||
return `${msg.timestamp}_${msg.senderPlatformId}_${(msg.content || '').length}`
|
||||
}
|
||||
|
||||
function getParsedMessageDisplayName(msg: ParsedMessage): string {
|
||||
return msg.senderGroupNickname || msg.senderAccountName || msg.senderPlatformId
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查消息是否是纯图片消息
|
||||
* 纯图片消息格式如:[图片: xxx.jpg]、[图片: {xxx}.jpg] 等
|
||||
@@ -191,7 +188,7 @@ function detectConflictsInMessages(
|
||||
if (conflicts.length < 5) {
|
||||
console.log(`[Merger] Conflict #${conflicts.length + 1}:`)
|
||||
console.log(` Timestamp: ${ts} (${new Date(ts * 1000).toLocaleString()})`)
|
||||
console.log(` Sender: ${sender} (${item1.msg.senderName})`)
|
||||
console.log(` Sender: ${sender} (${getParsedMessageDisplayName(item1.msg)})`)
|
||||
console.log(
|
||||
` File1: ${item1.source}, length: ${content1.length}, content: "${content1.slice(0, 50)}..."`
|
||||
)
|
||||
@@ -203,7 +200,7 @@ function detectConflictsInMessages(
|
||||
conflicts.push({
|
||||
id: `conflict_${ts}_${sender}_${conflicts.length}`,
|
||||
timestamp: ts,
|
||||
sender: item1.msg.senderName || sender,
|
||||
sender: getParsedMessageDisplayName(item1.msg) || sender,
|
||||
contentLength1: content1.length,
|
||||
contentLength2: content2.length,
|
||||
content1: content1,
|
||||
@@ -269,6 +266,128 @@ export async function mergeFilesWithCache(params: MergeParams, cache: Map<string
|
||||
}
|
||||
}
|
||||
|
||||
async function executeMerge(
|
||||
parseResults: Array<{ result: ParseResult; source: string }>,
|
||||
outputName: string,
|
||||
outputDir: string | undefined,
|
||||
_conflictResolutions: MergeParams['conflictResolutions'],
|
||||
andAnalyze: boolean
|
||||
): Promise<MergeResult> {
|
||||
const memberMap = new Map<string, ChatLabMember>()
|
||||
for (const { result } of parseResults) {
|
||||
for (const member of result.members) {
|
||||
const existing = memberMap.get(member.platformId)
|
||||
if (existing) {
|
||||
if (member.accountName) existing.accountName = member.accountName
|
||||
if (member.groupNickname) existing.groupNickname = member.groupNickname
|
||||
if (member.avatar) existing.avatar = member.avatar
|
||||
} else {
|
||||
memberMap.set(member.platformId, {
|
||||
platformId: member.platformId,
|
||||
accountName: member.accountName,
|
||||
groupNickname: member.groupNickname,
|
||||
avatar: member.avatar,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const seenKeys = new Set<string>()
|
||||
const mergedMessages: ChatLabMessage[] = []
|
||||
for (const { result } of parseResults) {
|
||||
for (const msg of result.messages) {
|
||||
const key = getMessageKey(msg)
|
||||
if (seenKeys.has(key)) continue
|
||||
seenKeys.add(key)
|
||||
|
||||
mergedMessages.push({
|
||||
sender: msg.senderPlatformId,
|
||||
accountName: msg.senderAccountName,
|
||||
groupNickname: msg.senderGroupNickname,
|
||||
timestamp: msg.timestamp,
|
||||
type: msg.type,
|
||||
content: msg.content,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
mergedMessages.sort((a, b) => a.timestamp - b.timestamp)
|
||||
|
||||
const sources: MergeSource[] = parseResults.map(({ result, source }) => ({
|
||||
filename: source,
|
||||
platform: result.meta.platform,
|
||||
messageCount: result.messages.length,
|
||||
}))
|
||||
|
||||
const groupIds = new Set(parseResults.map(({ result }) => result.meta.groupId).filter(Boolean))
|
||||
const groupId =
|
||||
groupIds.size === 1 ? parseResults.find(({ result }) => result.meta.groupId)?.result.meta.groupId : undefined
|
||||
const groupAvatar = groupId
|
||||
? parseResults.filter(({ result }) => result.meta.groupId === groupId).pop()?.result.meta.groupAvatar
|
||||
: undefined
|
||||
|
||||
const chatLabHeader = {
|
||||
version: '0.0.1',
|
||||
exportedAt: Math.floor(Date.now() / 1000),
|
||||
generator: 'ChatLab Merge Tool',
|
||||
description: `合并自 ${parseResults.length} 个文件`,
|
||||
}
|
||||
|
||||
const chatLabMeta = {
|
||||
name: outputName,
|
||||
platform: parseResults[0].result.meta.platform as ChatPlatform,
|
||||
type: parseResults[0].result.meta.type as ChatType,
|
||||
sources,
|
||||
groupId,
|
||||
groupAvatar,
|
||||
}
|
||||
|
||||
const targetDir = outputDir || getDefaultOutputDir()
|
||||
ensureOutputDir(targetDir)
|
||||
const outputPath = path.join(targetDir, generateOutputFilename(outputName, 'json'))
|
||||
|
||||
const chatLabData: ChatLabFormat = {
|
||||
chatlab: chatLabHeader,
|
||||
meta: chatLabMeta,
|
||||
members: Array.from(memberMap.values()),
|
||||
messages: mergedMessages,
|
||||
}
|
||||
fs.writeFileSync(outputPath, JSON.stringify(chatLabData, null, 2), 'utf-8')
|
||||
|
||||
let sessionId: string | undefined
|
||||
if (andAnalyze) {
|
||||
sessionId = importData({
|
||||
meta: {
|
||||
name: chatLabMeta.name,
|
||||
platform: chatLabMeta.platform,
|
||||
type: chatLabMeta.type,
|
||||
groupId: chatLabMeta.groupId,
|
||||
groupAvatar: chatLabMeta.groupAvatar,
|
||||
},
|
||||
members: chatLabData.members.map((member) => ({
|
||||
platformId: member.platformId,
|
||||
accountName: member.accountName,
|
||||
groupNickname: member.groupNickname,
|
||||
avatar: member.avatar,
|
||||
})),
|
||||
messages: chatLabData.messages.map((msg) => ({
|
||||
senderPlatformId: msg.sender,
|
||||
senderAccountName: msg.accountName,
|
||||
senderGroupNickname: msg.groupNickname,
|
||||
timestamp: msg.timestamp,
|
||||
type: msg.type,
|
||||
content: msg.content,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
outputPath,
|
||||
sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 临时数据库版本(内存优化) ====================
|
||||
|
||||
/**
|
||||
@@ -344,7 +463,14 @@ export async function mergeFilesWithTempDb(
|
||||
params: MergeParams,
|
||||
tempDbCache: Map<string, string>
|
||||
): Promise<MergeResult> {
|
||||
const { filePaths, outputName, outputDir, outputFormat = 'json', conflictResolutions, andAnalyze } = params
|
||||
const {
|
||||
filePaths,
|
||||
outputName,
|
||||
outputDir,
|
||||
outputFormat = 'json',
|
||||
conflictResolutions: _conflictResolutions,
|
||||
andAnalyze,
|
||||
} = params
|
||||
|
||||
console.log('[Merger] mergeFilesWithTempDb: Starting merge')
|
||||
console.log(
|
||||
@@ -685,17 +811,17 @@ export async function exportSessionToTempFile(sessionId: string): Promise<string
|
||||
},
|
||||
members: members.map((m) => ({
|
||||
platformId: m.platform_id,
|
||||
accountName: m.account_name,
|
||||
groupNickname: m.group_nickname,
|
||||
accountName: m.account_name || m.platform_id,
|
||||
groupNickname: m.group_nickname || undefined,
|
||||
avatar: m.avatar,
|
||||
})),
|
||||
messages: messages.map((msg) => ({
|
||||
sender: msg.sender,
|
||||
accountName: msg.accountName,
|
||||
groupNickname: msg.groupNickname,
|
||||
accountName: msg.accountName || msg.sender,
|
||||
groupNickname: msg.groupNickname || undefined,
|
||||
timestamp: msg.timestamp,
|
||||
type: msg.type,
|
||||
content: msg.content,
|
||||
type: msg.type as ChatLabMessage['type'],
|
||||
content: msg.content ?? null,
|
||||
})),
|
||||
}
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
import Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import type { ParsedMember, ParsedMessage } from '../../../src/types/base'
|
||||
import type { ParseResult, ParsedMeta } from '../parser/types'
|
||||
import { ChatType, type ParsedMember, type ParsedMessage } from '../../../src/types/base'
|
||||
import type { ParsedMeta } from '../parser/types'
|
||||
import { getTempDir as getAppTempDir, ensureDir } from '../paths'
|
||||
|
||||
/**
|
||||
@@ -201,7 +201,7 @@ export class TempDbReader {
|
||||
return {
|
||||
name: row.name,
|
||||
platform: row.platform,
|
||||
type: row.type as 'group' | 'private',
|
||||
type: row.type === ChatType.PRIVATE ? ChatType.PRIVATE : ChatType.GROUP,
|
||||
groupId: row.group_id || undefined,
|
||||
groupAvatar: row.group_avatar || undefined,
|
||||
}
|
||||
@@ -264,8 +264,8 @@ export class TempDbReader {
|
||||
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
||||
senderGroupNickname: r.sender_group_nickname || undefined,
|
||||
timestamp: r.timestamp,
|
||||
type: r.type,
|
||||
content: r.content || undefined,
|
||||
type: r.type as ParsedMessage['type'],
|
||||
content: r.content,
|
||||
}))
|
||||
|
||||
callback(messages)
|
||||
@@ -300,8 +300,8 @@ export class TempDbReader {
|
||||
senderAccountName: r.sender_account_name || r.sender_platform_id,
|
||||
senderGroupNickname: r.sender_group_nickname || undefined,
|
||||
timestamp: r.timestamp,
|
||||
type: r.type,
|
||||
content: r.content || undefined,
|
||||
type: r.type as ParsedMessage['type'],
|
||||
content: r.content,
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,11 @@ function getJieba(): JiebaInstance {
|
||||
throw new Error('jieba 模块加载失败')
|
||||
}
|
||||
}
|
||||
return jiebaInstance
|
||||
const instance = jiebaInstance
|
||||
if (!instance) {
|
||||
throw new Error('jieba 模块未初始化')
|
||||
}
|
||||
return instance
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
import * as fs from 'fs'
|
||||
import * as readline from 'readline'
|
||||
import * as path from 'path'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType, type ChatPlatform } from '../../../../src/types/base'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '../../../../src/types/base'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -29,13 +29,10 @@ import type {
|
||||
ParsedMember,
|
||||
ParsedMessage,
|
||||
} from '../types'
|
||||
import { getFileSize, createProgress, readFileHeadBytes } from '../utils'
|
||||
import { getFileSize, createProgress } from '../utils'
|
||||
|
||||
// ==================== JSONL 行类型定义 ====================
|
||||
|
||||
/** JSONL 行类型 */
|
||||
type JsonlLineType = 'header' | 'member' | 'message'
|
||||
|
||||
/** Header 行结构 */
|
||||
interface JsonlHeader {
|
||||
_type: 'header'
|
||||
|
||||
@@ -362,7 +362,7 @@ function cleanupTempFile(filePath: string): void {
|
||||
* QQ Chat Exporter 预处理器
|
||||
*/
|
||||
export const qqPreprocessor: Preprocessor = {
|
||||
needsPreprocess(filePath: string, fileSize: number): boolean {
|
||||
needsPreprocess(_filePath: string, fileSize: number): boolean {
|
||||
return fileSize > PREPROCESS_THRESHOLD
|
||||
},
|
||||
|
||||
|
||||
@@ -333,7 +333,7 @@ async function* parseDiscordExporter(options: ParseOptions): AsyncGenerator<Pars
|
||||
memberMap.set(author.id, {
|
||||
platformId: author.id,
|
||||
accountName: author.name,
|
||||
groupNickname: author.nickname || null,
|
||||
groupNickname: author.nickname || undefined,
|
||||
avatar: author.avatarUrl,
|
||||
roles: convertRoles(author.roles),
|
||||
})
|
||||
@@ -384,7 +384,7 @@ async function* parseDiscordExporter(options: ParseOptions): AsyncGenerator<Pars
|
||||
platformMessageId: msg.id, // 消息的平台原始 ID
|
||||
senderPlatformId: author.id,
|
||||
senderAccountName: author.name,
|
||||
senderGroupNickname: author.nickname || null,
|
||||
senderGroupNickname: author.nickname || undefined,
|
||||
timestamp: parseTimestamp(msg.timestamp),
|
||||
type: messageType,
|
||||
content: content || null,
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
ParseEvent,
|
||||
ParseResult,
|
||||
ParseProgress,
|
||||
LogLevel,
|
||||
FormatFeature,
|
||||
Parser,
|
||||
ParsedMeta,
|
||||
@@ -288,7 +289,7 @@ export interface StreamParseCallbacks {
|
||||
onMembers: (members: ParsedMember[]) => void
|
||||
onMessageBatch: (messages: ParsedMessage[]) => void
|
||||
/** 日志回调(可选) */
|
||||
onLog?: (level: 'info' | 'error', message: string) => void
|
||||
onLog?: (level: LogLevel, message: string) => void
|
||||
}
|
||||
|
||||
export interface StreamParseOptions extends StreamParseCallbacks {
|
||||
|
||||
@@ -24,7 +24,8 @@ export interface ParsedMeta {
|
||||
*/
|
||||
export interface ParseProgress {
|
||||
/** 阶段 */
|
||||
stage: 'detecting' | 'parsing' | 'done' | 'error'
|
||||
// 导入流程会复用解析进度结构,因此这里补充导入阶段枚举。
|
||||
stage: 'detecting' | 'parsing' | 'importing' | 'saving' | 'done' | 'error'
|
||||
/** 已读取字节数 */
|
||||
bytesRead: number
|
||||
/** 文件总字节数 */
|
||||
@@ -96,7 +97,7 @@ export interface FormatFeature {
|
||||
// ==================== 解析层:解析器接口 ====================
|
||||
|
||||
/** 日志级别 */
|
||||
export type LogLevel = 'info' | 'error'
|
||||
export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
|
||||
|
||||
/**
|
||||
* 解析选项
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*/
|
||||
|
||||
import * as fs from 'fs'
|
||||
import type { ParseProgress, CreateProgress } from './types'
|
||||
import type { CreateProgress } from './types'
|
||||
|
||||
/**
|
||||
* 获取文件大小
|
||||
|
||||
@@ -218,7 +218,7 @@ const checkUpdate = (win) => {
|
||||
})
|
||||
|
||||
// 不需要更新
|
||||
autoUpdater.on('update-not-available', (info) => {
|
||||
autoUpdater.on('update-not-available', (_info) => {
|
||||
// 客户端打开会默认弹一次,用isFirstShow来控制不弹
|
||||
if (isFirstShow) {
|
||||
isFirstShow = false
|
||||
@@ -283,7 +283,7 @@ const manualCheckForUpdates = () => {
|
||||
* 模拟更新弹窗(仅用于开发测试)
|
||||
* 控制台通过:window.api.app.simulateUpdate() 测试
|
||||
*/
|
||||
const simulateUpdateDialog = (win) => {
|
||||
const simulateUpdateDialog = (_win) => {
|
||||
dialog.showMessageBox({
|
||||
title: t('update.newVersionTitle', { version: '9.9.9' }),
|
||||
message: t('update.newVersionMessage', { version: '9.9.9' }),
|
||||
|
||||
@@ -227,6 +227,9 @@ export async function incrementalImport(
|
||||
if (processedCount % BATCH_SIZE === 0) {
|
||||
sendProgress(requestId, {
|
||||
stage: 'saving',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: processedCount,
|
||||
percentage: 50, // 实际进度难以计算,使用固定值
|
||||
message: `已处理 ${processedCount} 条,新增 ${newMessageCount} 条`,
|
||||
})
|
||||
@@ -244,6 +247,9 @@ export async function incrementalImport(
|
||||
|
||||
sendProgress(requestId, {
|
||||
stage: 'done',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: processedCount,
|
||||
percentage: 100,
|
||||
message: `导入完成,新增 ${newMessageCount} 条消息`,
|
||||
})
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
type ParsedMessage,
|
||||
type FormatFeature,
|
||||
} from '../../parser'
|
||||
import { ChatType } from '../../../../src/types/base'
|
||||
import { getDbDir } from '../core'
|
||||
import {
|
||||
initPerfLog,
|
||||
@@ -876,7 +877,7 @@ export async function streamParseFileInfo(filePath: string, requestId: string):
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
|
||||
let meta: ParsedMeta = { name: '未知群聊', platform: formatFeature.platform, type: 'group' }
|
||||
let meta: ParsedMeta = { name: '未知群聊', platform: formatFeature.platform, type: ChatType.GROUP }
|
||||
const memberSet = new Set<string>()
|
||||
let messageCount = 0
|
||||
let metaInserted = false
|
||||
|
||||
@@ -5,9 +5,6 @@
|
||||
|
||||
import { openDatabase } from '../core'
|
||||
|
||||
// 查询超时时间(毫秒)
|
||||
const QUERY_TIMEOUT_MS = 10000
|
||||
|
||||
/**
|
||||
* SQL 执行结果
|
||||
*/
|
||||
@@ -106,13 +103,6 @@ export function executePluginQuery<T = Record<string, any>>(
|
||||
return stmt.all(params) as T[]
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 SQL 是否包含 LIMIT 子句
|
||||
*/
|
||||
function hasLimit(sql: string): boolean {
|
||||
return /\bLIMIT\s+\d+/i.test(sql)
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行用户 SQL 查询
|
||||
* - 只支持 SELECT 语句
|
||||
|
||||
@@ -7,6 +7,7 @@ import { Worker } from 'worker_threads'
|
||||
import { app } from 'electron'
|
||||
import * as path from 'path'
|
||||
import type { ParseProgress } from '../parser'
|
||||
import type { StreamImportResult } from './import'
|
||||
import { getDatabaseDir, ensureDir } from '../paths'
|
||||
|
||||
// Worker 实例
|
||||
@@ -333,10 +334,11 @@ export async function closeDatabase(sessionId: string): Promise<void> {
|
||||
export interface MemberWithStats {
|
||||
id: number
|
||||
platformId: string
|
||||
name: string
|
||||
nickname: string | null
|
||||
accountName: string | null
|
||||
groupNickname: string | null
|
||||
aliases: string[]
|
||||
messageCount: number
|
||||
avatar?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -410,7 +412,7 @@ export async function streamImport(
|
||||
filePath: string,
|
||||
onProgress?: (progress: ParseProgress) => void,
|
||||
formatOptions?: Record<string, unknown>
|
||||
): Promise<{ success: boolean; sessionId?: string; error?: string }> {
|
||||
): Promise<StreamImportResult> {
|
||||
return sendToWorkerWithProgress('streamImport', { filePath, formatOptions }, onProgress)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user