mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
feat(import): orchestrate automatic imports
This commit is contained in:
1 parent
27f46dfcb6
commit
6cf4c063fc
4 files changed
+289
No files matched your search
@@ -0,0 +1,143 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { autoImportFile, type AutoImportDeps } from './auto-importer'
|
||||
import type { AutoImportDecision } from './auto-import-matcher'
|
||||
|
||||
function createDeps(options?: {
|
||||
existingSessionIds?: string[]
|
||||
decision?: AutoImportDecision
|
||||
matchError?: Error
|
||||
createdSessionId?: string
|
||||
incrementalNewCount?: number
|
||||
incrementalDuplicateCount?: number
|
||||
}) {
|
||||
const calls = {
|
||||
match: 0,
|
||||
create: [] as Array<{ filePath: string; sessionId?: string }>,
|
||||
append: [] as Array<{ sessionId: string; filePath: string }>,
|
||||
}
|
||||
const existing = new Set(options?.existingSessionIds ?? [])
|
||||
|
||||
const deps: AutoImportDeps = {
|
||||
listSessionIds: () => [...existing],
|
||||
openReadonly: () => {
|
||||
throw new Error('not used by fake matcher')
|
||||
},
|
||||
sessionExists: (sessionId) => existing.has(sessionId),
|
||||
resolveTarget: async () => {
|
||||
calls.match++
|
||||
if (options?.matchError) throw options.matchError
|
||||
return options?.decision ?? { action: 'create', reason: 'no-match' }
|
||||
},
|
||||
createSession: async (filePath, _formatOptions, sessionId) => {
|
||||
calls.create.push({ filePath, sessionId })
|
||||
return {
|
||||
success: true,
|
||||
sessionId: sessionId ?? options?.createdSessionId ?? 'created-session',
|
||||
diagnostics: {
|
||||
logFile: null,
|
||||
detectedFormat: 'fixture',
|
||||
messagesReceived: 12,
|
||||
messagesWritten: 10,
|
||||
messagesSkipped: 2,
|
||||
skipReasons: { noSenderId: 0, noAccountName: 0, invalidTimestamp: 2, noType: 0 },
|
||||
},
|
||||
}
|
||||
},
|
||||
appendSession: async (sessionId, filePath) => {
|
||||
calls.append.push({ sessionId, filePath })
|
||||
const newMessageCount = options?.incrementalNewCount ?? 3
|
||||
return {
|
||||
success: true,
|
||||
newMessageCount,
|
||||
batch: {
|
||||
receivedCount: newMessageCount + (options?.incrementalDuplicateCount ?? 5),
|
||||
writtenCount: newMessageCount,
|
||||
duplicateCount: options?.incrementalDuplicateCount ?? 5,
|
||||
errorCount: 0,
|
||||
errorReasonCounts: {},
|
||||
errorSample: [],
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return { deps, calls }
|
||||
}
|
||||
|
||||
test('explicit existing session forces incremental import without matching', async () => {
|
||||
const { deps, calls } = createDeps({ existingSessionIds: ['explicit'] })
|
||||
|
||||
const result = await autoImportFile('source.json', deps, { explicitSessionId: 'explicit' })
|
||||
|
||||
assert.deepEqual(result, {
|
||||
success: true,
|
||||
sessionId: 'explicit',
|
||||
importMode: 'incremental',
|
||||
newMessageCount: 3,
|
||||
duplicateCount: 5,
|
||||
})
|
||||
assert.equal(calls.match, 0)
|
||||
assert.deepEqual(calls.create, [])
|
||||
assert.deepEqual(calls.append, [{ sessionId: 'explicit', filePath: 'source.json' }])
|
||||
})
|
||||
|
||||
test('explicit missing session creates with the requested id without matching', async () => {
|
||||
const { deps, calls } = createDeps()
|
||||
|
||||
const result = await autoImportFile('source.json', deps, { explicitSessionId: 'requested' })
|
||||
|
||||
assert.equal(result.success, true)
|
||||
assert.equal(result.sessionId, 'requested')
|
||||
assert.equal(result.importMode, 'created')
|
||||
assert.equal(result.newMessageCount, 10)
|
||||
assert.equal(result.duplicateCount, 0)
|
||||
assert.equal(calls.match, 0)
|
||||
assert.deepEqual(calls.create, [{ filePath: 'source.json', sessionId: 'requested' }])
|
||||
})
|
||||
|
||||
test('automatic unique match preserves incremental mode when every message is duplicate', async () => {
|
||||
const { deps, calls } = createDeps({
|
||||
decision: { action: 'incremental', sessionId: 'existing', matchedBy: 'trailing-messages' },
|
||||
incrementalNewCount: 0,
|
||||
incrementalDuplicateCount: 500,
|
||||
})
|
||||
|
||||
const result = await autoImportFile('source.json', deps)
|
||||
|
||||
assert.deepEqual(result, {
|
||||
success: true,
|
||||
sessionId: 'existing',
|
||||
importMode: 'incremental',
|
||||
matchedBy: 'trailing-messages',
|
||||
newMessageCount: 0,
|
||||
duplicateCount: 500,
|
||||
})
|
||||
assert.equal(calls.match, 1)
|
||||
assert.deepEqual(calls.create, [])
|
||||
})
|
||||
|
||||
for (const reason of ['no-match', 'ambiguous'] as const) {
|
||||
test(`automatic ${reason} decision creates a new session`, async () => {
|
||||
const { deps, calls } = createDeps({ decision: { action: 'create', reason } })
|
||||
|
||||
const result = await autoImportFile('source.json', deps)
|
||||
|
||||
assert.equal(result.success, true)
|
||||
assert.equal(result.importMode, 'created')
|
||||
assert.equal(calls.match, 1)
|
||||
assert.equal(calls.create.length, 1)
|
||||
assert.deepEqual(calls.append, [])
|
||||
})
|
||||
}
|
||||
|
||||
test('matcher failure stops import instead of silently creating a session', async () => {
|
||||
const { deps, calls } = createDeps({ matchError: new Error('candidate database failed') })
|
||||
|
||||
const result = await autoImportFile('source.json', deps)
|
||||
|
||||
assert.equal(result.success, false)
|
||||
assert.match(result.error ?? '', /candidate database failed/)
|
||||
assert.deepEqual(calls.create, [])
|
||||
assert.deepEqual(calls.append, [])
|
||||
})
|
||||
@@ -0,0 +1,140 @@
|
||||
import { appLogger } from '../logging/app-logger'
|
||||
import {
|
||||
resolveAutoImportTarget,
|
||||
type AutoImportDecision,
|
||||
type AutoImportMatcherDeps,
|
||||
type AutoImportMatchMethod,
|
||||
} from './auto-import-matcher'
|
||||
import type { IncrementalImportResult } from './incremental-importer'
|
||||
import type { ImportDiagnostics, StreamImportResult } from './streaming-importer'
|
||||
|
||||
export interface AutoImportOptions {
|
||||
explicitSessionId?: string
|
||||
formatOptions?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface AutoImportDeps extends AutoImportMatcherDeps {
|
||||
sessionExists(sessionId: string): boolean
|
||||
createSession(
|
||||
filePath: string,
|
||||
formatOptions?: Record<string, unknown>,
|
||||
sessionId?: string
|
||||
): Promise<StreamImportResult>
|
||||
appendSession(
|
||||
sessionId: string,
|
||||
filePath: string,
|
||||
formatOptions?: Record<string, unknown>
|
||||
): Promise<IncrementalImportResult>
|
||||
resolveTarget?: typeof resolveAutoImportTarget
|
||||
}
|
||||
|
||||
export interface AutoImportResult {
|
||||
success: boolean
|
||||
sessionId?: string
|
||||
importMode?: 'created' | 'incremental'
|
||||
matchedBy?: AutoImportMatchMethod
|
||||
newMessageCount?: number
|
||||
duplicateCount?: number
|
||||
diagnostics?: ImportDiagnostics
|
||||
error?: string
|
||||
}
|
||||
|
||||
function mapCreateResult(result: StreamImportResult): AutoImportResult {
|
||||
if (!result.success || !result.sessionId) {
|
||||
return { success: false, error: result.error, diagnostics: result.diagnostics }
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
sessionId: result.sessionId,
|
||||
importMode: 'created',
|
||||
newMessageCount: result.diagnostics?.messagesWritten ?? 0,
|
||||
duplicateCount: 0,
|
||||
diagnostics: result.diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
function mapIncrementalResult(
|
||||
sessionId: string,
|
||||
result: IncrementalImportResult,
|
||||
matchedBy?: AutoImportMatchMethod
|
||||
): AutoImportResult {
|
||||
if (!result.success) return { success: false, sessionId, error: result.error }
|
||||
return {
|
||||
success: true,
|
||||
sessionId,
|
||||
importMode: 'incremental',
|
||||
...(matchedBy ? { matchedBy } : {}),
|
||||
newMessageCount: result.newMessageCount,
|
||||
duplicateCount: result.batch?.duplicateCount ?? 0,
|
||||
}
|
||||
}
|
||||
|
||||
export async function autoImportFile(
|
||||
filePath: string,
|
||||
deps: AutoImportDeps,
|
||||
options: AutoImportOptions = {}
|
||||
): Promise<AutoImportResult> {
|
||||
try {
|
||||
if (options.explicitSessionId) {
|
||||
if (deps.sessionExists(options.explicitSessionId)) {
|
||||
const result = mapIncrementalResult(
|
||||
options.explicitSessionId,
|
||||
await deps.appendSession(options.explicitSessionId, filePath, options.formatOptions)
|
||||
)
|
||||
appLogger.info('import', 'Explicit incremental import completed', {
|
||||
sessionId: options.explicitSessionId,
|
||||
success: result.success,
|
||||
newMessageCount: result.newMessageCount,
|
||||
duplicateCount: result.duplicateCount,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
const result = mapCreateResult(
|
||||
await deps.createSession(filePath, options.formatOptions, options.explicitSessionId)
|
||||
)
|
||||
appLogger.info('import', 'Explicit session import completed', {
|
||||
sessionId: result.sessionId,
|
||||
success: result.success,
|
||||
importMode: result.importMode,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
appLogger.info('import', 'Automatic session matching started', {
|
||||
candidateCount: deps.listSessionIds().length,
|
||||
})
|
||||
const decision: AutoImportDecision = await (deps.resolveTarget ?? resolveAutoImportTarget)(
|
||||
filePath,
|
||||
deps,
|
||||
options.formatOptions
|
||||
)
|
||||
|
||||
if (decision.action === 'incremental') {
|
||||
const result = mapIncrementalResult(
|
||||
decision.sessionId,
|
||||
await deps.appendSession(decision.sessionId, filePath, options.formatOptions),
|
||||
decision.matchedBy
|
||||
)
|
||||
appLogger.info('import', 'Automatic incremental import completed', {
|
||||
sessionId: decision.sessionId,
|
||||
matchedBy: decision.matchedBy,
|
||||
success: result.success,
|
||||
newMessageCount: result.newMessageCount,
|
||||
duplicateCount: result.duplicateCount,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
const result = mapCreateResult(await deps.createSession(filePath, options.formatOptions))
|
||||
appLogger.info('import', 'Automatic import created a new session', {
|
||||
reason: decision.reason,
|
||||
sessionId: result.sessionId,
|
||||
success: result.success,
|
||||
})
|
||||
return result
|
||||
} catch (error) {
|
||||
appLogger.error('import', 'Automatic import failed', error)
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) }
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,8 @@ export type {
|
||||
} from './incremental-importer'
|
||||
export { resolveAutoImportTarget } from './auto-import-matcher'
|
||||
export type { AutoImportDecision, AutoImportMatcherDeps, AutoImportMatchMethod } from './auto-import-matcher'
|
||||
export { autoImportFile } from './auto-importer'
|
||||
export type { AutoImportDeps, AutoImportOptions, AutoImportResult } from './auto-importer'
|
||||
export { ZipArchiveReader, validateArchiveEntryName } from './archive/archive-reader'
|
||||
export { ArchiveImportError } from './archive/errors'
|
||||
export { GoogleChatTakeoutResolver } from './archive/google-chat-resolver'
|
||||
|
||||
@@ -16,6 +16,7 @@ export {
|
||||
analyzeIncrementalImport,
|
||||
incrementalImport,
|
||||
resolveAutoImportTarget,
|
||||
autoImportFile,
|
||||
logNativeParserStatus,
|
||||
ZipArchiveReader,
|
||||
validateArchiveEntryName,
|
||||
@@ -42,6 +43,9 @@ export type {
|
||||
AutoImportDecision,
|
||||
AutoImportMatcherDeps,
|
||||
AutoImportMatchMethod,
|
||||
AutoImportDeps,
|
||||
AutoImportOptions,
|
||||
AutoImportResult,
|
||||
ArchiveEntrySummary,
|
||||
ArchiveEntryStreamOpener,
|
||||
ArchiveEntryVisitor,
|
||||
|
||||
Reference in new issue
Block a user