mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
fix(parser): preprocess large native fallbacks
Retry strict native fallback through the slim QQ export path so compatible large imports retain the existing memory safeguard.
This commit is contained in:
1 parent
5d943ab0b3
commit
2e8ca2bebd
3 files changed
+70
-8
No files matched your search
@@ -42,7 +42,7 @@ async function* pumpNativeParser(
|
||||
adapter: NativeFormatAdapter,
|
||||
options: ParseOptions,
|
||||
parser: NativeParser,
|
||||
fallback: ParseGenerator
|
||||
fallbackAfterNativeFailure: ParseGenerator
|
||||
): AsyncGenerator<ParseEvent, void, unknown> {
|
||||
const { filePath, batchSize = 5000, onProgress, onLog } = options
|
||||
|
||||
@@ -76,7 +76,7 @@ async function* pumpNativeParser(
|
||||
`[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
// Only progress events have been emitted so far — safe to restart with TS.
|
||||
yield* fallback(options)
|
||||
yield* fallbackAfterNativeFailure(options)
|
||||
return
|
||||
} finally {
|
||||
clearInterval(progressTimer)
|
||||
@@ -112,7 +112,11 @@ async function* pumpNativeParser(
|
||||
}
|
||||
|
||||
/** Wrap a TS parse generator with native acceleration for the given format. */
|
||||
export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback: ParseGenerator): ParseGenerator {
|
||||
export function createNativeFirstParser(
|
||||
adapter: NativeFormatAdapter,
|
||||
fallback: ParseGenerator,
|
||||
fallbackAfterNativeFailure: ParseGenerator = fallback
|
||||
): ParseGenerator {
|
||||
return async function* (options: ParseOptions): AsyncGenerator<ParseEvent, void, unknown> {
|
||||
const native = loadNativeParser()
|
||||
if (!native) {
|
||||
@@ -136,6 +140,6 @@ export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback:
|
||||
return
|
||||
}
|
||||
|
||||
yield* pumpNativeParser(adapter, options, parser, fallback)
|
||||
yield* pumpNativeParser(adapter, options, parser, fallbackAfterNativeFailure)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
import { KNOWN_PLATFORMS, ChatType, type MessageType } from '@openchatlab/shared-types'
|
||||
import type { NativeMember, NativeMessage } from '@openchatlab/parser-native'
|
||||
import type { ParsedMember, ParsedMessage, ParsedMeta } from '../types'
|
||||
import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types'
|
||||
import { getFileSize } from '../utils'
|
||||
import { shuakamiQqPreprocessor } from '../formats/shuakami-qq-preprocessor'
|
||||
import { createNativeFirstParser, type NativeFormatAdapter, type ParseGenerator } from './create-native-parser'
|
||||
|
||||
interface ShuakamiQqMetaJson {
|
||||
@@ -58,7 +60,40 @@ export const shuakamiQqAdapter: NativeFormatAdapter = {
|
||||
},
|
||||
}
|
||||
|
||||
async function* fallbackAfterNativeFailure(
|
||||
fallback: ParseGenerator,
|
||||
options: ParseOptions
|
||||
): AsyncGenerator<ParseEvent, void, unknown> {
|
||||
if (!shuakamiQqPreprocessor.needsPreprocess(options.filePath, getFileSize(options.filePath))) {
|
||||
yield* fallback(options)
|
||||
return
|
||||
}
|
||||
|
||||
let slimFilePath: string | null = null
|
||||
try {
|
||||
options.onLog?.('info', '[NativeParser] Preprocessing large export before TypeScript fallback')
|
||||
slimFilePath = await shuakamiQqPreprocessor.preprocess(options.filePath, options.onProgress)
|
||||
} catch (error) {
|
||||
options.onLog?.(
|
||||
'warn',
|
||||
`[NativeParser] Fallback preprocessing failed; retrying the original export: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
yield* fallback(options)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
yield* fallback({ ...options, filePath: slimFilePath })
|
||||
} finally {
|
||||
if (slimFilePath) shuakamiQqPreprocessor.cleanup(slimFilePath)
|
||||
}
|
||||
}
|
||||
|
||||
/** Wrap the pure TypeScript shuakami/qq-chat-exporter V4 parser with N-API acceleration. */
|
||||
export function withShuakamiQqNative(fallback: ParseGenerator): ParseGenerator {
|
||||
return createNativeFirstParser(shuakamiQqAdapter, fallback)
|
||||
return createNativeFirstParser(shuakamiQqAdapter, fallback, (options) =>
|
||||
fallbackAfterNativeFailure(fallback, options)
|
||||
)
|
||||
}
|
||||
Reference in new issue
Block a user