fix(parser): preprocess large native fallbacks

Retry strict native fallback through the slim QQ export path so compatible large imports retain the existing memory safeguard.
This commit is contained in:
digua committed 2026-07-30 21:01:20 +08:00
1 parent 5d943ab0b3
commit 2e8ca2bebd
3 files changed
+70 -8

No files matched your search

@@ -42,7 +42,7 @@ async function* pumpNativeParser(
adapter: NativeFormatAdapter,
options: ParseOptions,
parser: NativeParser,
fallback: ParseGenerator
fallbackAfterNativeFailure: ParseGenerator
): AsyncGenerator<ParseEvent, void, unknown> {
const { filePath, batchSize = 5000, onProgress, onLog } = options
@@ -76,7 +76,7 @@ async function* pumpNativeParser(
`[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}`
)
// Only progress events have been emitted so far — safe to restart with TS.
yield* fallback(options)
yield* fallbackAfterNativeFailure(options)
return
} finally {
clearInterval(progressTimer)
@@ -112,7 +112,11 @@ async function* pumpNativeParser(
}
/** Wrap a TS parse generator with native acceleration for the given format. */
export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback: ParseGenerator): ParseGenerator {
export function createNativeFirstParser(
adapter: NativeFormatAdapter,
fallback: ParseGenerator,
fallbackAfterNativeFailure: ParseGenerator = fallback
): ParseGenerator {
return async function* (options: ParseOptions): AsyncGenerator<ParseEvent, void, unknown> {
const native = loadNativeParser()
if (!native) {
@@ -136,6 +140,6 @@ export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback:
return
}
yield* pumpNativeParser(adapter, options, parser, fallback)
yield* pumpNativeParser(adapter, options, parser, fallbackAfterNativeFailure)
}
}
@@ -2,7 +2,9 @@
import { KNOWN_PLATFORMS, ChatType, type MessageType } from '@openchatlab/shared-types'
import type { NativeMember, NativeMessage } from '@openchatlab/parser-native'
import type { ParsedMember, ParsedMessage, ParsedMeta } from '../types'
import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types'
import { getFileSize } from '../utils'
import { shuakamiQqPreprocessor } from '../formats/shuakami-qq-preprocessor'
import { createNativeFirstParser, type NativeFormatAdapter, type ParseGenerator } from './create-native-parser'
interface ShuakamiQqMetaJson {
@@ -58,7 +60,40 @@ export const shuakamiQqAdapter: NativeFormatAdapter = {
},
}
async function* fallbackAfterNativeFailure(
fallback: ParseGenerator,
options: ParseOptions
): AsyncGenerator<ParseEvent, void, unknown> {
if (!shuakamiQqPreprocessor.needsPreprocess(options.filePath, getFileSize(options.filePath))) {
yield* fallback(options)
return
}
let slimFilePath: string | null = null
try {
options.onLog?.('info', '[NativeParser] Preprocessing large export before TypeScript fallback')
slimFilePath = await shuakamiQqPreprocessor.preprocess(options.filePath, options.onProgress)
} catch (error) {
options.onLog?.(
'warn',
`[NativeParser] Fallback preprocessing failed; retrying the original export: ${
error instanceof Error ? error.message : String(error)
}`
)
yield* fallback(options)
return
}
try {
yield* fallback({ ...options, filePath: slimFilePath })
} finally {
if (slimFilePath) shuakamiQqPreprocessor.cleanup(slimFilePath)
}
}
/** Wrap the pure TypeScript shuakami/qq-chat-exporter V4 parser with N-API acceleration. */
export function withShuakamiQqNative(fallback: ParseGenerator): ParseGenerator {
return createNativeFirstParser(shuakamiQqAdapter, fallback)
return createNativeFirstParser(shuakamiQqAdapter, fallback, (options) =>
fallbackAfterNativeFailure(fallback, options)
)
}