From 2e8ca2bebd545f3db987110cdc9e35ceeeb8c6f8 Mon Sep 17 00:00:00 2001 From: digua Date: Thu, 30 Jul 2026 20:40:56 +0800 Subject: [PATCH] fix(parser): preprocess large native fallbacks Retry strict native fallback through the slim QQ export path so compatible large imports retain the existing memory safeguard. --- .../src/import/streaming-importer.test.ts | 27 ++++++++++++- .../parser/src/native/create-native-parser.ts | 12 ++++-- .../parser/src/native/shuakami-qq-native.ts | 39 ++++++++++++++++++- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/node-runtime/src/import/streaming-importer.test.ts b/packages/node-runtime/src/import/streaming-importer.test.ts index ab166ffd..ac04d9fe 100644 --- a/packages/node-runtime/src/import/streaming-importer.test.ts +++ b/packages/node-runtime/src/import/streaming-importer.test.ts @@ -190,7 +190,7 @@ function writeSingleFileShuakamiQqExport(root: string): string { return filePath } -function writeLargeSingleFileShuakamiQqExport(root: string): string { +function writeLargeSingleFileShuakamiQqExport(root: string, timestamp = '2026-07-10T12:00:00.000Z'): string { const filePath = path.join(root, 'shuakami-qq-v4-large.json') const descriptor = fs.openSync(filePath, 'w') try { @@ -202,7 +202,7 @@ function writeLargeSingleFileShuakamiQqExport(root: string): string { statistics: { senders: [{ uid: 'u_10001' }] }, messages: [], }).slice(0, -2) + - '{"messageId":"large-message","timestamp":"2026-07-10T12:00:00.000Z",' + + `{"messageId":"large-message","timestamp":${JSON.stringify(timestamp)},` + '"sender":{"uin":"10001","name":"Alice"},"content":{"text":"large native message","html":"' ) const oneMegabyte = 'H'.repeat(1024 * 1024) @@ -464,6 +464,29 @@ test( } ) +test( + 'streamingImport preprocesses a >50MB shuakami/qq-chat-exporter export after strict native fallback', + { + skip: !isNativeFormatAvailable('shuakami-qq-exporter') && 'native shuakami/qq-chat-exporter kernel not built', + }, + async (t) => { + const root = makeTempDir() + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const filePath = writeLargeSingleFileShuakamiQqExport(root, 'July 10, 2026 12:00:00 GMT') + const dbPath = path.join(root, 'shuakami-qq-native-large-fallback.db') + const logMessages: string[] = [] + const deps = createImportDeps(dbPath) + deps.logger = createCollectingLogger(logMessages) + + const result = await streamingImport(filePath, deps, undefined, 'shuakami-qq-native-large-fallback') + + assert.equal(result.success, true) + assert.equal(result.diagnostics?.messagesWritten, 1) + assert.ok(logMessages.some((message) => message.includes('falling back to TS parser'))) + assert.ok(logMessages.some((message) => message.includes('Preprocessing large export before TypeScript fallback'))) + } +) + test('streamingImport cleans a generated shuakami/qq-chat-exporter slim file when database setup fails', async (t) => { const root = makeTempDir() t.after(() => fs.rmSync(root, { recursive: true, force: true })) diff --git a/packages/parser/src/native/create-native-parser.ts b/packages/parser/src/native/create-native-parser.ts index bbfc589a..07dbf473 100644 --- a/packages/parser/src/native/create-native-parser.ts +++ b/packages/parser/src/native/create-native-parser.ts @@ -42,7 +42,7 @@ async function* pumpNativeParser( adapter: NativeFormatAdapter, options: ParseOptions, parser: NativeParser, - fallback: ParseGenerator + fallbackAfterNativeFailure: ParseGenerator ): AsyncGenerator { const { filePath, batchSize = 5000, onProgress, onLog } = options @@ -76,7 +76,7 @@ async function* pumpNativeParser( `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` ) // Only progress events have been emitted so far — safe to restart with TS. - yield* fallback(options) + yield* fallbackAfterNativeFailure(options) return } finally { clearInterval(progressTimer) @@ -112,7 +112,11 @@ async function* pumpNativeParser( } /** Wrap a TS parse generator with native acceleration for the given format. */ -export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback: ParseGenerator): ParseGenerator { +export function createNativeFirstParser( + adapter: NativeFormatAdapter, + fallback: ParseGenerator, + fallbackAfterNativeFailure: ParseGenerator = fallback +): ParseGenerator { return async function* (options: ParseOptions): AsyncGenerator { const native = loadNativeParser() if (!native) { @@ -136,6 +140,6 @@ export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback: return } - yield* pumpNativeParser(adapter, options, parser, fallback) + yield* pumpNativeParser(adapter, options, parser, fallbackAfterNativeFailure) } } diff --git a/packages/parser/src/native/shuakami-qq-native.ts b/packages/parser/src/native/shuakami-qq-native.ts index b135064a..74addd12 100644 --- a/packages/parser/src/native/shuakami-qq-native.ts +++ b/packages/parser/src/native/shuakami-qq-native.ts @@ -2,7 +2,9 @@ import { KNOWN_PLATFORMS, ChatType, type MessageType } from '@openchatlab/shared-types' import type { NativeMember, NativeMessage } from '@openchatlab/parser-native' -import type { ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import { getFileSize } from '../utils' +import { shuakamiQqPreprocessor } from '../formats/shuakami-qq-preprocessor' import { createNativeFirstParser, type NativeFormatAdapter, type ParseGenerator } from './create-native-parser' interface ShuakamiQqMetaJson { @@ -58,7 +60,40 @@ export const shuakamiQqAdapter: NativeFormatAdapter = { }, } +async function* fallbackAfterNativeFailure( + fallback: ParseGenerator, + options: ParseOptions +): AsyncGenerator { + if (!shuakamiQqPreprocessor.needsPreprocess(options.filePath, getFileSize(options.filePath))) { + yield* fallback(options) + return + } + + let slimFilePath: string | null = null + try { + options.onLog?.('info', '[NativeParser] Preprocessing large export before TypeScript fallback') + slimFilePath = await shuakamiQqPreprocessor.preprocess(options.filePath, options.onProgress) + } catch (error) { + options.onLog?.( + 'warn', + `[NativeParser] Fallback preprocessing failed; retrying the original export: ${ + error instanceof Error ? error.message : String(error) + }` + ) + yield* fallback(options) + return + } + + try { + yield* fallback({ ...options, filePath: slimFilePath }) + } finally { + if (slimFilePath) shuakamiQqPreprocessor.cleanup(slimFilePath) + } +} + /** Wrap the pure TypeScript shuakami/qq-chat-exporter V4 parser with N-API acceleration. */ export function withShuakamiQqNative(fallback: ParseGenerator): ParseGenerator { - return createNativeFirstParser(shuakamiQqAdapter, fallback) + return createNativeFirstParser(shuakamiQqAdapter, fallback, (options) => + fallbackAfterNativeFailure(fallback, options) + ) }