refactor: 清理 parser worker rag merger 的历史类型问题

This commit is contained in:
digua
2026-03-25 22:43:12 +08:00
parent b6fdc3887e
commit 7eaba396ec
29 changed files with 264 additions and 103 deletions
+5 -4
View File
@@ -5,7 +5,7 @@
import { getActiveConfig, buildPiModel } from '../llm'
import { getAllTools, createActivateSkillTool } from '../tools'
import type { ToolContext, OwnerInfo } from '../tools/types'
import type { ToolContext } from '../tools/types'
import { getHistoryForAgent } from '../conversations'
import { aiLogger, isDebugMode } from '../logger'
import { t as i18nT } from '../../i18n'
@@ -16,7 +16,7 @@ import {
type Usage as PiUsage,
} from '@mariozechner/pi-ai'
import type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, SkillContext } from './types'
import type { AgentConfig, AgentStreamChunk, AgentResult, SkillContext } from './types'
import type { AssistantConfig } from '../assistant/types'
import { buildSystemPrompt } from './prompt-builder'
import { extractThinkingContent, stripToolCallTags } from './content-parser'
@@ -25,7 +25,7 @@ import { AgentEventHandler } from './event-handler'
type SimpleHistoryMessage = { role: 'user' | 'assistant'; content: string }
// Re-export types for external consumers
export type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, AgentRuntimeStatus } from './types'
export type { AgentConfig, AgentStreamChunk, AgentResult, TokenUsage, AgentRuntimeStatus, SkillContext } from './types'
/**
* Agent 执行器类
@@ -120,7 +120,8 @@ export class Agent {
if (newMessages.length > 0) {
const parts: string[] = []
for (const m of newMessages) {
const msg = m as Record<string, unknown>
// pi-ai 的 Message 联合类型较严格,这里仅用于调试日志读取动态字段。
const msg = m as unknown as Record<string, unknown>
parts.push(`--- ${msg.role} ---`)
const content = msg.content as
| Array<{ type: string; text?: string; name?: string; arguments?: unknown }>
+19 -7
View File
@@ -5,8 +5,9 @@
* 通过 pluginQuery 执行参数化 SQL 并格式化结果。
*/
import { Type, type TObject, type TProperties } from '@mariozechner/pi-ai'
import type { AgentTool } from '@mariozechner/pi-agent-core'
import { Type } from '@mariozechner/pi-ai'
import type { AgentTool, AgentToolResult } from '@mariozechner/pi-agent-core'
import type { TSchema } from '@sinclair/typebox'
import type { ToolContext } from '../tools/types'
import type { CustomSqlToolDef, JsonSchemaObject } from './types'
import * as workerManager from '../../worker/workerManager'
@@ -17,8 +18,8 @@ import { t as i18nT } from '../../i18n'
*
* 仅覆盖 SQL 工具参数定义的常见类型(string / number / integer / boolean)。
*/
export function jsonSchemaToTypeBox(schema: JsonSchemaObject): TObject<TProperties> {
const props: TProperties = {}
export function jsonSchemaToTypeBox(schema: JsonSchemaObject) {
const props: Record<string, TSchema> = {}
for (const [key, prop] of Object.entries(schema.properties)) {
const isRequired = schema.required?.includes(key) ?? false
@@ -77,13 +78,21 @@ export function createSqlTool(def: CustomSqlToolDef, context: ToolContext): Agen
label: def.name,
description: def.description,
parameters: schema,
execute: async (_toolCallId: string, params: Record<string, unknown>) => {
execute: async (
_toolCallId: string,
params: Record<string, unknown>,
_signal?: AbortSignal,
_onUpdate?: unknown
): Promise<AgentToolResult<{ rows: Record<string, unknown>[]; rowCount: number }>> => {
const rows = await workerManager.pluginQuery(context.sessionId, def.execution.query, params)
const fallback = resolveTemplate(def.name, 'fallback', def.execution.fallback)
if (!rows || rows.length === 0) {
return { content: [{ type: 'text' as const, text: fallback }] }
return {
content: [{ type: 'text' as const, text: fallback }],
details: { rows: [], rowCount: 0 },
}
}
const rowTemplate = resolveTemplate(def.name, 'rowTemplate', def.execution.rowTemplate)
@@ -102,7 +111,10 @@ export function createSqlTool(def: CustomSqlToolDef, context: ToolContext): Agen
lines.push(formatRow(rowTemplate, row as Record<string, unknown>))
}
return { content: [{ type: 'text' as const, text: lines.join('\n') }] }
return {
content: [{ type: 'text' as const, text: lines.join('\n') }],
details: { rows, rowCount: rows.length },
}
},
}
}
+1 -1
View File
@@ -5,7 +5,7 @@
import * as fs from 'fs'
import * as path from 'path'
import { getLogsDir, ensureDir } from '../paths'
import { getLogsDir } from '../paths'
let debugMode = false
+2 -2
View File
@@ -308,7 +308,7 @@ export function getSessionChunks(dbPath: string, options: ChunkingOptions = {}):
return chunks
} catch (error) {
logger.error('[Chunking] Failed to get session chunks:', error)
logger.error('Chunking', 'Failed to get session chunks', error)
return []
} finally {
if (db) {
@@ -372,7 +372,7 @@ export function getSessionChunk(dbPath: string, sessionId: number): Chunk | null
},
}
} catch (error) {
logger.error('[Chunking] Failed to get single session chunk:', error)
logger.error('Chunking', 'Failed to get single session chunk', error)
return null
} finally {
if (db) {
+3 -2
View File
@@ -62,7 +62,7 @@ async function rewriteQuery(query: string, abortSignal?: AbortSignal): Promise<s
return rewritten || query
} catch (error) {
logger.warn('[Semantic Pipeline] Query rewrite failed, using original query:', error)
logger.warn('Semantic Pipeline', 'Query rewrite failed, using original query', error)
return query
}
}
@@ -197,7 +197,8 @@ export async function executeSemanticPipeline(options: SemanticPipelineOptions):
chunkVectors.set(chunk.id, vector)
if (vectorStore) {
await vectorStore.add(chunk.id, vector, chunk.metadata as Record<string, unknown>)
// ChunkMetadata 是结构化对象,这里仅在向量存储层按通用元数据字典处理。
await vectorStore.add(chunk.id, vector, chunk.metadata as unknown as Record<string, unknown>)
}
}
}
+4 -4
View File
@@ -32,7 +32,7 @@ export async function getVectorStore(): Promise<IVectorStore | null> {
activeStore = await createVectorStore(config.vectorStore)
return activeStore
} catch (error) {
logger.error('[Store Manager] Failed to create store:', error)
logger.error('Store Manager', 'Failed to create store', error)
return null
}
}
@@ -44,13 +44,13 @@ async function createVectorStore(config: VectorStoreConfig): Promise<IVectorStor
switch (config.type) {
case 'memory': {
const capacity = config.memoryCacheSize || 10000
logger.info(`[Store Manager] Using memory store, capacity: ${capacity}`)
logger.info('Store Manager', `Using memory store, capacity: ${capacity}`)
return new MemoryVectorStore(capacity)
}
case 'sqlite': {
const dbPath = config.dbPath || path.join(getVectorStoreDir(), 'embeddings.db')
logger.info(`[Store Manager] Using SQLite store: ${dbPath}`)
logger.info('Store Manager', `Using SQLite store: ${dbPath}`)
return new SQLiteVectorStore(dbPath)
}
@@ -72,7 +72,7 @@ export async function resetVectorStore(): Promise<void> {
if (activeStore) {
await activeStore.close()
activeStore = null
logger.info('[Store Manager] Store reset')
logger.info('Store Manager', 'Store reset')
}
}
+2 -2
View File
@@ -224,7 +224,7 @@ export class MemoryVectorStore implements IVectorStore {
this.cache.clear()
this.head = null
this.tail = null
logger.info('[Memory Store] All vectors cleared')
logger.info('Memory Store', 'All vectors cleared')
}
/**
@@ -258,6 +258,6 @@ export class MemoryVectorStore implements IVectorStore {
*/
async close(): Promise<void> {
// 内存存储无需关闭操作
logger.info('[Memory Store] Closed')
logger.info('Memory Store', 'Closed')
}
}
+6 -5
View File
@@ -79,7 +79,7 @@ export class SQLiteVectorStore implements IVectorStore {
// 索引可能已存在
}
logger.info(`[SQLite Store] Initialized: ${this.dbPath}`)
logger.info('SQLite Store', `Initialized: ${this.dbPath}`)
}
/**
@@ -107,8 +107,9 @@ export class SQLiteVectorStore implements IVectorStore {
VALUES (?, ?, ?, ?)
`)
const insertMany = this.db.transaction((items: typeof items) => {
for (const item of items) {
type VectorBatchItem = { id: string; vector: number[]; metadata?: Record<string, unknown> }
const insertMany = this.db.transaction((batchItems: VectorBatchItem[]) => {
for (const item of batchItems) {
const buffer = vectorToBuffer(item.vector)
insert.run(item.id, buffer, item.vector.length, item.metadata ? JSON.stringify(item.metadata) : null)
}
@@ -180,7 +181,7 @@ export class SQLiteVectorStore implements IVectorStore {
*/
async clear(): Promise<void> {
this.db.exec('DELETE FROM vectors')
logger.info('[SQLite Store] All vectors cleared')
logger.info('SQLite Store', 'All vectors cleared')
}
/**
@@ -214,6 +215,6 @@ export class SQLiteVectorStore implements IVectorStore {
*/
async close(): Promise<void> {
this.db.close()
logger.info('[SQLite Store] Closed')
logger.info('SQLite Store', 'Closed')
}
}
+15 -6
View File
@@ -142,7 +142,7 @@ function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): Agen
const originalExecute = tool.execute
return {
...tool,
execute: async (toolCallId: string, params: any) => {
execute: async (toolCallId: string, params: any, _signal?: AbortSignal, _onUpdate?: unknown) => {
const result = await originalExecute(toolCallId, params)
const details = result.details as Record<string, unknown> | undefined
@@ -160,8 +160,8 @@ function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): Agen
const formatted = processed.map((m) => formatMessageCompact(m, context.locale))
const finalDetails = { ...details, messages: formatted, returned: processed.length }
delete finalDetails.rawMessages
const { rawMessages: _rawMessages, ...restDetails } = details
const finalDetails = { ...restDetails, messages: formatted, returned: processed.length }
let textContent = formatToolResultAsText(finalDetails)
if (nameMapLine) {
@@ -242,6 +242,7 @@ export function createActivateSkillTool(
return {
name: 'activate_skill',
label: 'activate_skill',
description: isZh
? '激活一个分析技能,获取该技能的详细执行指导'
: 'Activate an analysis skill and get its detailed execution instructions',
@@ -255,11 +256,12 @@ export function createActivateSkillTool(
},
required: ['skill_id'],
},
execute: async (_toolCallId: string, params: { skill_id: string }) => {
execute: async (_toolCallId: string, params: { skill_id: string }, _signal?: AbortSignal, _onUpdate?: unknown) => {
const skill: SkillDef | null = getSkillConfig(params.skill_id)
if (!skill) {
return {
content: [{ type: 'text' as const, text: isZh ? '技能不存在' : 'Skill not found' }],
details: { skillId: params.skill_id, found: false },
}
}
@@ -267,7 +269,10 @@ export function createActivateSkillTool(
const scopeMsg = isZh
? `该技能仅适用于${skill.chatScope === 'group' ? '群聊' : '私聊'}场景`
: `This skill is only applicable to ${skill.chatScope === 'group' ? 'group chat' : 'private chat'} scenarios`
return { content: [{ type: 'text' as const, text: scopeMsg }] }
return {
content: [{ type: 'text' as const, text: scopeMsg }],
details: { skillId: params.skill_id, found: true, applicable: false },
}
}
if (skill.tools.length > 0 && allowedTools && allowedTools.length > 0) {
@@ -276,7 +281,10 @@ export function createActivateSkillTool(
const msg = isZh
? `当前助手缺少该技能所需的工具:${missing.join(', ')}`
: `Current assistant lacks tools required by this skill: ${missing.join(', ')}`
return { content: [{ type: 'text' as const, text: msg }] }
return {
content: [{ type: 'text' as const, text: msg }],
details: { skillId: params.skill_id, found: true, applicable: false, missingTools: missing },
}
}
}
@@ -286,6 +294,7 @@ export function createActivateSkillTool(
return {
content: [{ type: 'text' as const, text: `${skill.prompt}${actionPrompt}` }],
details: { skillId: params.skill_id, found: true, applicable: true },
}
},
}
+3 -1
View File
@@ -26,7 +26,9 @@ export const i18nTexts = {
},
}
export function t(key: keyof typeof i18nTexts, locale?: string): string | string[] {
type TextEntryKey = Exclude<keyof typeof i18nTexts, 'dailySummary'>
export function t(key: TextEntryKey, locale?: string): string | string[] {
const text = i18nTexts[key]
if (typeof text === 'object' && 'zh' in text && 'en' in text) {
return isChineseLocale(locale) ? text.zh : text.en
+17 -6
View File
@@ -328,6 +328,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
originators: [],
initiators: [],
breakers: [],
fastestRepeaters: [],
originatorRates: [],
initiatorRates: [],
breakerRates: [],
@@ -389,16 +390,16 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
const contentStats = new Map<
string,
{ count: number; maxChainLength: number; originatorId: number; lastTs: number }
{ count: number; maxChainLength: number; originatorId: number; lastTs: number; firstMessageId: number }
>()
let currentContent: string | null = null
let repeatChain: Array<{ senderId: number; content: string; ts: number }> = []
let repeatChain: Array<{ id: number; senderId: number; content: string; ts: number }> = []
let totalRepeatChains = 0
let totalChainLength = 0
const processRepeatChain = (
chain: Array<{ senderId: number; content: string; ts: number }>,
chain: Array<{ id: number; senderId: number; content: string; ts: number }>,
breakerId?: number
) => {
if (chain.length < 3) return
@@ -421,6 +422,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
const content = chain[0].content
const chainTs = chain[0].ts
const firstMessageId = chain[0].id
const existing = contentStats.get(content)
if (existing) {
existing.count++
@@ -428,9 +430,16 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
if (chainLength > existing.maxChainLength) {
existing.maxChainLength = chainLength
existing.originatorId = originatorId
existing.firstMessageId = firstMessageId
}
} else {
contentStats.set(content, { count: 1, maxChainLength: chainLength, originatorId, lastTs: chainTs })
contentStats.set(content, {
count: 1,
maxChainLength: chainLength,
originatorId,
lastTs: chainTs,
firstMessageId,
})
}
}
@@ -446,13 +455,13 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
if (content === currentContent) {
const lastSender = repeatChain[repeatChain.length - 1]?.senderId
if (lastSender !== msg.senderId) {
repeatChain.push({ senderId: msg.senderId, content, ts: msg.ts })
repeatChain.push({ id: msg.id, senderId: msg.senderId, content, ts: msg.ts })
}
} else {
processRepeatChain(repeatChain, msg.senderId)
currentContent = content
repeatChain = [{ senderId: msg.senderId, content, ts: msg.ts }]
repeatChain = [{ id: msg.id, senderId: msg.senderId, content, ts: msg.ts }]
}
}
@@ -509,6 +518,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
maxChainLength: stats.maxChainLength,
originatorName: originatorInfo?.name || '未知',
lastTs: stats.lastTs,
firstMessageId: stats.firstMessageId,
})
}
hotContents.sort((a, b) => b.maxChainLength - a.maxChainLength)
@@ -518,6 +528,7 @@ export function getRepeatAnalysis(sessionId: string, filter?: TimeFilter): Repea
originators: buildRankList(originatorCount, totalRepeatChains),
initiators: buildRankList(initiatorCount, totalRepeatChains),
breakers: buildRankList(breakerCount, totalRepeatChains),
fastestRepeaters: [],
originatorRates: buildRateList(originatorCount),
initiatorRates: buildRateList(initiatorCount),
breakerRates: buildRateList(breakerCount),
+1 -7
View File
@@ -6,7 +6,7 @@
import Database from 'better-sqlite3'
import * as fs from 'fs'
import * as path from 'path'
import type { DbMeta, ParseResult, AnalysisSession } from '../../../src/types/base'
import type { ParseResult } from '../../../src/types/base'
import { migrateDatabase, needsMigration, CURRENT_SCHEMA_VERSION } from './migrations'
import { getDatabaseDir, ensureDir } from '../paths'
@@ -162,7 +162,6 @@ export function openDatabaseWithMigration(sessionId: string, forceRepair = false
*/
export function importData(parseResult: ParseResult): string {
const sessionId = generateSessionId()
const dbPath = getDbPath(sessionId)
const db = createDatabase(sessionId)
try {
@@ -293,17 +292,12 @@ export function importData(parseResult: ParseResult): string {
})
importTransaction()
const fileExists = fs.existsSync(dbPath)
return sessionId
} catch (error) {
console.error('[Database] Error in importData:', error)
throw error
} finally {
db.close()
const fileExists = fs.existsSync(dbPath)
}
}
+1 -1
View File
@@ -237,7 +237,7 @@ class MainProcess {
})
// 监听渲染进程崩溃
app.on('render-process-gone', (e, w, d) => {
app.on('render-process-gone', (_event, w, d) => {
if (d.reason == 'crashed') {
w.reload()
}
+1 -1
View File
@@ -8,7 +8,7 @@ import * as worker from '../worker/workerManager'
import * as parser from '../parser'
import { detectFormat, diagnoseFormat, scanMultiChatFile, type ParseProgress } from '../parser'
import type { IpcContext } from './types'
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos, type MigrationInfo } from '../database/migrations'
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos } from '../database/migrations'
import { exportSessionToTempFile, cleanupTempExportFiles } from '../merger'
import { t } from '../i18n'
+143 -17
View File
@@ -9,21 +9,14 @@ import { parseFileSync, detectFormat } from '../parser'
import { importData } from '../database/core'
import { TempDbReader } from './tempCache'
import { getDownloadsDir } from '../paths'
import type { ParseResult, ParsedMessage, ChatPlatform, ChatType, ParsedMember } from '../../../src/types/base'
import type {
ParseResult,
ParsedMessage,
ChatLabFormat,
ChatLabMember,
ChatLabMessage,
FileParseInfo,
MergeConflict,
ChatPlatform,
ChatType,
ParsedMember,
} from '../../../src/types/base'
import type {
ConflictCheckResult,
ConflictResolution,
MergeParams,
MergeResult,
MergeSource,
@@ -83,6 +76,10 @@ function getMessageKey(msg: ParsedMessage): string {
return `${msg.timestamp}_${msg.senderPlatformId}_${(msg.content || '').length}`
}
function getParsedMessageDisplayName(msg: ParsedMessage): string {
return msg.senderGroupNickname || msg.senderAccountName || msg.senderPlatformId
}
/**
* 检查消息是否是纯图片消息
* 纯图片消息格式如:[图片: xxx.jpg]、[图片: {xxx}.jpg] 等
@@ -191,7 +188,7 @@ function detectConflictsInMessages(
if (conflicts.length < 5) {
console.log(`[Merger] Conflict #${conflicts.length + 1}:`)
console.log(` Timestamp: ${ts} (${new Date(ts * 1000).toLocaleString()})`)
console.log(` Sender: ${sender} (${item1.msg.senderName})`)
console.log(` Sender: ${sender} (${getParsedMessageDisplayName(item1.msg)})`)
console.log(
` File1: ${item1.source}, length: ${content1.length}, content: "${content1.slice(0, 50)}..."`
)
@@ -203,7 +200,7 @@ function detectConflictsInMessages(
conflicts.push({
id: `conflict_${ts}_${sender}_${conflicts.length}`,
timestamp: ts,
sender: item1.msg.senderName || sender,
sender: getParsedMessageDisplayName(item1.msg) || sender,
contentLength1: content1.length,
contentLength2: content2.length,
content1: content1,
@@ -269,6 +266,128 @@ export async function mergeFilesWithCache(params: MergeParams, cache: Map<string
}
}
async function executeMerge(
parseResults: Array<{ result: ParseResult; source: string }>,
outputName: string,
outputDir: string | undefined,
_conflictResolutions: MergeParams['conflictResolutions'],
andAnalyze: boolean
): Promise<MergeResult> {
const memberMap = new Map<string, ChatLabMember>()
for (const { result } of parseResults) {
for (const member of result.members) {
const existing = memberMap.get(member.platformId)
if (existing) {
if (member.accountName) existing.accountName = member.accountName
if (member.groupNickname) existing.groupNickname = member.groupNickname
if (member.avatar) existing.avatar = member.avatar
} else {
memberMap.set(member.platformId, {
platformId: member.platformId,
accountName: member.accountName,
groupNickname: member.groupNickname,
avatar: member.avatar,
})
}
}
}
const seenKeys = new Set<string>()
const mergedMessages: ChatLabMessage[] = []
for (const { result } of parseResults) {
for (const msg of result.messages) {
const key = getMessageKey(msg)
if (seenKeys.has(key)) continue
seenKeys.add(key)
mergedMessages.push({
sender: msg.senderPlatformId,
accountName: msg.senderAccountName,
groupNickname: msg.senderGroupNickname,
timestamp: msg.timestamp,
type: msg.type,
content: msg.content,
})
}
}
mergedMessages.sort((a, b) => a.timestamp - b.timestamp)
const sources: MergeSource[] = parseResults.map(({ result, source }) => ({
filename: source,
platform: result.meta.platform,
messageCount: result.messages.length,
}))
const groupIds = new Set(parseResults.map(({ result }) => result.meta.groupId).filter(Boolean))
const groupId =
groupIds.size === 1 ? parseResults.find(({ result }) => result.meta.groupId)?.result.meta.groupId : undefined
const groupAvatar = groupId
? parseResults.filter(({ result }) => result.meta.groupId === groupId).pop()?.result.meta.groupAvatar
: undefined
const chatLabHeader = {
version: '0.0.1',
exportedAt: Math.floor(Date.now() / 1000),
generator: 'ChatLab Merge Tool',
description: `合并自 ${parseResults.length} 个文件`,
}
const chatLabMeta = {
name: outputName,
platform: parseResults[0].result.meta.platform as ChatPlatform,
type: parseResults[0].result.meta.type as ChatType,
sources,
groupId,
groupAvatar,
}
const targetDir = outputDir || getDefaultOutputDir()
ensureOutputDir(targetDir)
const outputPath = path.join(targetDir, generateOutputFilename(outputName, 'json'))
const chatLabData: ChatLabFormat = {
chatlab: chatLabHeader,
meta: chatLabMeta,
members: Array.from(memberMap.values()),
messages: mergedMessages,
}
fs.writeFileSync(outputPath, JSON.stringify(chatLabData, null, 2), 'utf-8')
let sessionId: string | undefined
if (andAnalyze) {
sessionId = importData({
meta: {
name: chatLabMeta.name,
platform: chatLabMeta.platform,
type: chatLabMeta.type,
groupId: chatLabMeta.groupId,
groupAvatar: chatLabMeta.groupAvatar,
},
members: chatLabData.members.map((member) => ({
platformId: member.platformId,
accountName: member.accountName,
groupNickname: member.groupNickname,
avatar: member.avatar,
})),
messages: chatLabData.messages.map((msg) => ({
senderPlatformId: msg.sender,
senderAccountName: msg.accountName,
senderGroupNickname: msg.groupNickname,
timestamp: msg.timestamp,
type: msg.type,
content: msg.content,
})),
})
}
return {
success: true,
outputPath,
sessionId,
}
}
// ==================== 临时数据库版本(内存优化) ====================
/**
@@ -344,7 +463,14 @@ export async function mergeFilesWithTempDb(
params: MergeParams,
tempDbCache: Map<string, string>
): Promise<MergeResult> {
const { filePaths, outputName, outputDir, outputFormat = 'json', conflictResolutions, andAnalyze } = params
const {
filePaths,
outputName,
outputDir,
outputFormat = 'json',
conflictResolutions: _conflictResolutions,
andAnalyze,
} = params
console.log('[Merger] mergeFilesWithTempDb: Starting merge')
console.log(
@@ -685,17 +811,17 @@ export async function exportSessionToTempFile(sessionId: string): Promise<string
},
members: members.map((m) => ({
platformId: m.platform_id,
accountName: m.account_name,
groupNickname: m.group_nickname,
accountName: m.account_name || m.platform_id,
groupNickname: m.group_nickname || undefined,
avatar: m.avatar,
})),
messages: messages.map((msg) => ({
sender: msg.sender,
accountName: msg.accountName,
groupNickname: msg.groupNickname,
accountName: msg.accountName || msg.sender,
groupNickname: msg.groupNickname || undefined,
timestamp: msg.timestamp,
type: msg.type,
content: msg.content,
type: msg.type as ChatLabMessage['type'],
content: msg.content ?? null,
})),
}
+7 -7
View File
@@ -6,8 +6,8 @@
import Database from 'better-sqlite3'
import * as fs from 'fs'
import * as path from 'path'
import type { ParsedMember, ParsedMessage } from '../../../src/types/base'
import type { ParseResult, ParsedMeta } from '../parser/types'
import { ChatType, type ParsedMember, type ParsedMessage } from '../../../src/types/base'
import type { ParsedMeta } from '../parser/types'
import { getTempDir as getAppTempDir, ensureDir } from '../paths'
/**
@@ -201,7 +201,7 @@ export class TempDbReader {
return {
name: row.name,
platform: row.platform,
type: row.type as 'group' | 'private',
type: row.type === ChatType.PRIVATE ? ChatType.PRIVATE : ChatType.GROUP,
groupId: row.group_id || undefined,
groupAvatar: row.group_avatar || undefined,
}
@@ -264,8 +264,8 @@ export class TempDbReader {
senderAccountName: r.sender_account_name || r.sender_platform_id,
senderGroupNickname: r.sender_group_nickname || undefined,
timestamp: r.timestamp,
type: r.type,
content: r.content || undefined,
type: r.type as ParsedMessage['type'],
content: r.content,
}))
callback(messages)
@@ -300,8 +300,8 @@ export class TempDbReader {
senderAccountName: r.sender_account_name || r.sender_platform_id,
senderGroupNickname: r.sender_group_nickname || undefined,
timestamp: r.timestamp,
type: r.type,
content: r.content || undefined,
type: r.type as ParsedMessage['type'],
content: r.content,
}))
}
+5 -1
View File
@@ -32,7 +32,11 @@ function getJieba(): JiebaInstance {
throw new Error('jieba 模块加载失败')
}
}
return jiebaInstance
const instance = jiebaInstance
if (!instance) {
throw new Error('jieba 模块未初始化')
}
return instance
}
/**
@@ -18,7 +18,7 @@
import * as fs from 'fs'
import * as readline from 'readline'
import * as path from 'path'
import { KNOWN_PLATFORMS, ChatType, MessageType, type ChatPlatform } from '../../../../src/types/base'
import { KNOWN_PLATFORMS, ChatType, MessageType } from '../../../../src/types/base'
import type {
FormatFeature,
FormatModule,
@@ -29,13 +29,10 @@ import type {
ParsedMember,
ParsedMessage,
} from '../types'
import { getFileSize, createProgress, readFileHeadBytes } from '../utils'
import { getFileSize, createProgress } from '../utils'
// ==================== JSONL 行类型定义 ====================
/** JSONL 行类型 */
type JsonlLineType = 'header' | 'member' | 'message'
/** Header 行结构 */
interface JsonlHeader {
_type: 'header'
@@ -362,7 +362,7 @@ function cleanupTempFile(filePath: string): void {
* QQ Chat Exporter 预处理器
*/
export const qqPreprocessor: Preprocessor = {
needsPreprocess(filePath: string, fileSize: number): boolean {
needsPreprocess(_filePath: string, fileSize: number): boolean {
return fileSize > PREPROCESS_THRESHOLD
},
@@ -333,7 +333,7 @@ async function* parseDiscordExporter(options: ParseOptions): AsyncGenerator<Pars
memberMap.set(author.id, {
platformId: author.id,
accountName: author.name,
groupNickname: author.nickname || null,
groupNickname: author.nickname || undefined,
avatar: author.avatarUrl,
roles: convertRoles(author.roles),
})
@@ -384,7 +384,7 @@ async function* parseDiscordExporter(options: ParseOptions): AsyncGenerator<Pars
platformMessageId: msg.id, // 消息的平台原始 ID
senderPlatformId: author.id,
senderAccountName: author.name,
senderGroupNickname: author.nickname || null,
senderGroupNickname: author.nickname || undefined,
timestamp: parseTimestamp(msg.timestamp),
type: messageType,
content: content || null,
+2 -1
View File
@@ -11,6 +11,7 @@ import type {
ParseEvent,
ParseResult,
ParseProgress,
LogLevel,
FormatFeature,
Parser,
ParsedMeta,
@@ -288,7 +289,7 @@ export interface StreamParseCallbacks {
onMembers: (members: ParsedMember[]) => void
onMessageBatch: (messages: ParsedMessage[]) => void
/** 日志回调(可选) */
onLog?: (level: 'info' | 'error', message: string) => void
onLog?: (level: LogLevel, message: string) => void
}
export interface StreamParseOptions extends StreamParseCallbacks {
+3 -2
View File
@@ -24,7 +24,8 @@ export interface ParsedMeta {
*/
export interface ParseProgress {
/** 阶段 */
stage: 'detecting' | 'parsing' | 'done' | 'error'
// 导入流程会复用解析进度结构,因此这里补充导入阶段枚举。
stage: 'detecting' | 'parsing' | 'importing' | 'saving' | 'done' | 'error'
/** 已读取字节数 */
bytesRead: number
/** 文件总字节数 */
@@ -96,7 +97,7 @@ export interface FormatFeature {
// ==================== 解析层:解析器接口 ====================
/** 日志级别 */
export type LogLevel = 'info' | 'error'
export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
/**
* 解析选项
+1 -1
View File
@@ -3,7 +3,7 @@
*/
import * as fs from 'fs'
import type { ParseProgress, CreateProgress } from './types'
import type { CreateProgress } from './types'
/**
* 获取文件大小
+2 -2
View File
@@ -218,7 +218,7 @@ const checkUpdate = (win) => {
})
// 不需要更新
autoUpdater.on('update-not-available', (info) => {
autoUpdater.on('update-not-available', (_info) => {
// 客户端打开会默认弹一次,用isFirstShow来控制不弹
if (isFirstShow) {
isFirstShow = false
@@ -283,7 +283,7 @@ const manualCheckForUpdates = () => {
* 模拟更新弹窗(仅用于开发测试)
* 控制台通过:window.api.app.simulateUpdate() 测试
*/
const simulateUpdateDialog = (win) => {
const simulateUpdateDialog = (_win) => {
dialog.showMessageBox({
title: t('update.newVersionTitle', { version: '9.9.9' }),
message: t('update.newVersionMessage', { version: '9.9.9' }),
@@ -227,6 +227,9 @@ export async function incrementalImport(
if (processedCount % BATCH_SIZE === 0) {
sendProgress(requestId, {
stage: 'saving',
bytesRead: 0,
totalBytes: 0,
messagesProcessed: processedCount,
percentage: 50, // 实际进度难以计算,使用固定值
message: `已处理 ${processedCount} 条,新增 ${newMessageCount}`,
})
@@ -244,6 +247,9 @@ export async function incrementalImport(
sendProgress(requestId, {
stage: 'done',
bytesRead: 0,
totalBytes: 0,
messagesProcessed: processedCount,
percentage: 100,
message: `导入完成,新增 ${newMessageCount} 条消息`,
})
+2 -1
View File
@@ -17,6 +17,7 @@ import {
type ParsedMessage,
type FormatFeature,
} from '../../parser'
import { ChatType } from '../../../../src/types/base'
import { getDbDir } from '../core'
import {
initPerfLog,
@@ -876,7 +877,7 @@ export async function streamParseFileInfo(filePath: string, requestId: string):
VALUES (?, ?, ?, ?, ?, ?)
`)
let meta: ParsedMeta = { name: '未知群聊', platform: formatFeature.platform, type: 'group' }
let meta: ParsedMeta = { name: '未知群聊', platform: formatFeature.platform, type: ChatType.GROUP }
const memberSet = new Set<string>()
let messageCount = 0
let metaInserted = false
-10
View File
@@ -5,9 +5,6 @@
import { openDatabase } from '../core'
// 查询超时时间(毫秒)
const QUERY_TIMEOUT_MS = 10000
/**
* SQL 执行结果
*/
@@ -106,13 +103,6 @@ export function executePluginQuery<T = Record<string, any>>(
return stmt.all(params) as T[]
}
/**
* 检查 SQL 是否包含 LIMIT 子句
*/
function hasLimit(sql: string): boolean {
return /\bLIMIT\s+\d+/i.test(sql)
}
/**
* 执行用户 SQL 查询
* - 只支持 SELECT 语句
+5 -3
View File
@@ -7,6 +7,7 @@ import { Worker } from 'worker_threads'
import { app } from 'electron'
import * as path from 'path'
import type { ParseProgress } from '../parser'
import type { StreamImportResult } from './import'
import { getDatabaseDir, ensureDir } from '../paths'
// Worker 实例
@@ -333,10 +334,11 @@ export async function closeDatabase(sessionId: string): Promise<void> {
export interface MemberWithStats {
id: number
platformId: string
name: string
nickname: string | null
accountName: string | null
groupNickname: string | null
aliases: string[]
messageCount: number
avatar?: string | null
}
/**
@@ -410,7 +412,7 @@ export async function streamImport(
filePath: string,
onProgress?: (progress: ParseProgress) => void,
formatOptions?: Record<string, unknown>
): Promise<{ success: boolean; sessionId?: string; error?: string }> {
): Promise<StreamImportResult> {
return sendToWorkerWithProgress('streamImport', { filePath, formatOptions }, onProgress)
}