diff --git a/apps/cli/src/http/routes/web/import.ts b/apps/cli/src/http/routes/web/import.ts index 88faabc7..50820713 100644 --- a/apps/cli/src/http/routes/web/import.ts +++ b/apps/cli/src/http/routes/web/import.ts @@ -12,6 +12,7 @@ import { importDemoSessions, type DatabaseManager, } from '@openchatlab/node-runtime' +import { PARSER_FORMAT_IDS } from '@openchatlab/parser' import { autoImport, autoImportBatch, @@ -84,7 +85,7 @@ export function registerImportRoutes( options.runPreparedImport ?? (async (manifestPath: string, onProgress: (progress: unknown) => void, sessionGapThreshold?: number) => { const result = await runAutoImport(manifestPath, { - formatId: 'google-chat-takeout', + formatId: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE, sessionGapThreshold, nativeBinding: resolveNativeBinding(), onProgress: onProgress as any, diff --git a/apps/desktop/main/ipc/chat.ts b/apps/desktop/main/ipc/chat.ts index e4480700..6243beaf 100644 --- a/apps/desktop/main/ipc/chat.ts +++ b/apps/desktop/main/ipc/chat.ts @@ -11,6 +11,7 @@ import { appLogger } from '@openchatlab/node-runtime' import * as databaseCore from '../database/core' import * as worker from '../worker/workerManager' import { + PARSER_FORMAT_IDS, detectFormat, findEntryFileInDirectory, getSupportedFormats, @@ -155,7 +156,7 @@ export function registerChatHandlers(ctx: IpcContext): void { worker.autoImport( manifestPath, forwardImportProgress, - { formatId: 'google-chat-takeout' }, + { formatId: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE }, undefined, options?.sessionGapThreshold ) diff --git a/packages/node-runtime/src/import/archive/google-chat-resolver.ts b/packages/node-runtime/src/import/archive/google-chat-resolver.ts index 2711aa22..20b76c7a 100644 --- a/packages/node-runtime/src/import/archive/google-chat-resolver.ts +++ b/packages/node-runtime/src/import/archive/google-chat-resolver.ts @@ -5,6 +5,7 @@ import streamChain from 'stream-chain' import streamJson from 'stream-json' import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' +import { PARSER_FORMAT_IDS } from '@openchatlab/parser' import { ArchiveImportError } from './errors' import type { ZipArchiveReader } from './archive-reader' import type { ArchiveEntrySummary, ArchiveResolver, MaterializedImport, PreparedImportChat } from './types' @@ -88,7 +89,7 @@ function deriveConversationName(draft: ConversationDraft, ownerEmail: string | n } export class GoogleChatTakeoutResolver implements ArchiveResolver { - readonly id = 'google-chat-takeout' + readonly id = PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE readonly platform = 'google-chat' detect(entries: ArchiveEntrySummary[]): boolean { diff --git a/packages/node-runtime/src/import/streaming-importer.test.ts b/packages/node-runtime/src/import/streaming-importer.test.ts index 625c761b..bdef9bd8 100644 --- a/packages/node-runtime/src/import/streaming-importer.test.ts +++ b/packages/node-runtime/src/import/streaming-importer.test.ts @@ -459,7 +459,7 @@ test('streamingImport builds the session index exactly once with the requested g test( 'streamingImport persists shuakami/qq-chat-exporter V4 native-first output without a derived FTS table', { - skip: !isNativeFormatAvailable('shuakami-qq-exporter') && 'native shuakami/qq-chat-exporter kernel not built', + skip: !isNativeFormatAvailable('qq-shuakami') && 'native shuakami/qq-chat-exporter kernel not built', }, async (t) => { const root = makeTempDir() @@ -529,7 +529,7 @@ test('streamingImport skips preprocessing when the selected format has native su isNativeFormatAvailable: () => true, } - const result = await streamingImport(filePath, deps, { formatId: 'shuakami-qq-exporter' }, 'native-preprocess-gate') + const result = await streamingImport(filePath, deps, { formatId: 'qq-shuakami' }, 'native-preprocess-gate') assert.equal(result.success, true) assert.equal(result.diagnostics?.messagesWritten, 2) @@ -538,7 +538,7 @@ test('streamingImport skips preprocessing when the selected format has native su assert.equal(fixture.getTempFilePath(), null) }) -test('streamingImport preprocesses directly when native support is unavailable', async (t) => { +test('streamingImport accepts a released format alias when native support is unavailable', async (t) => { const root = makeTempDir() t.after(() => fs.rmSync(root, { recursive: true, force: true })) const filePath = writeSingleFileShuakamiQqExport(root) @@ -586,7 +586,7 @@ test('streamingImport cleans a generated shuakami/qq-chat-exporter slim file whe isNativeFormatAvailable: () => false, }, }, - { formatId: 'shuakami-qq-exporter' }, + { formatId: 'qq-shuakami' }, 'shuakami-qq-preprocess-setup-failure' ), /database setup failed for test/ diff --git a/packages/parser-native/src/lib.rs b/packages/parser-native/src/lib.rs index 1432e527..065e473b 100644 --- a/packages/parser-native/src/lib.rs +++ b/packages/parser-native/src/lib.rs @@ -31,7 +31,7 @@ use protocol::{NativeMember, NativeParseProgress}; // ==================== Format registry ==================== /// Kernel ids understood by the `NativeParser` constructor. The TS side maps -/// sniffer format ids onto these (e.g. ycccccccy-echotrace reuses "weflow"). +/// public parser ids onto these (e.g. echotrace reuses "weflow"). #[derive(Clone, Copy)] enum FormatKind { Weflow, diff --git a/packages/parser/browser.ts b/packages/parser/browser.ts index ed6c66e0..ad51a814 100644 --- a/packages/parser/browser.ts +++ b/packages/parser/browser.ts @@ -25,3 +25,10 @@ export { type WeFlowJsonParseResult, } from './src/browser/weflow' export { rebaseChatLabDemoDocuments, type RebasedChatLabDemoDocuments } from './src/chatlab-demo-timeline' +export { + PARSER_FORMAT_IDS, + LEGACY_PARSER_FORMAT_ID_ALIASES, + isParserFormatId, + normalizeParserFormatId, + type ParserFormatId, +} from './src/format-ids' diff --git a/packages/parser/src/format-ids.test.ts b/packages/parser/src/format-ids.test.ts new file mode 100644 index 00000000..3899381e --- /dev/null +++ b/packages/parser/src/format-ids.test.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { + getFormatFeatureById, + getSupportedFormats, + LEGACY_PARSER_FORMAT_ID_ALIASES, + normalizeParserFormatId, +} from './index' + +describe('parser format ID compatibility', () => { + it('advertises canonical IDs while resolving every released legacy input alias', () => { + const supportedIds = new Set(getSupportedFormats().map((format) => format.id)) + + for (const [legacyId, canonicalId] of Object.entries(LEGACY_PARSER_FORMAT_ID_ALIASES)) { + assert.equal(normalizeParserFormatId(legacyId), canonicalId) + assert.equal(getFormatFeatureById(legacyId)?.id, canonicalId) + assert.equal(supportedIds.has(canonicalId), true) + assert.equal(supportedIds.has(legacyId), false) + } + }) + + it('leaves unknown IDs unchanged so callers can reject them normally', () => { + assert.equal(normalizeParserFormatId('future-format'), 'future-format') + assert.equal(getFormatFeatureById('future-format'), null) + }) +}) diff --git a/packages/parser/src/format-ids.ts b/packages/parser/src/format-ids.ts new file mode 100644 index 00000000..0767b2bf --- /dev/null +++ b/packages/parser/src/format-ids.ts @@ -0,0 +1,37 @@ +export const PARSER_FORMAT_IDS = { + CHATLAB: 'chatlab', + CHATLAB_JSONL: 'chatlab-jsonl', + QQ_SHUAKAMI: 'qq-shuakami', + QQ_SHUAKAMI_CHUNKED: 'qq-shuakami-chunked', + WEFLOW: 'weflow', + ECHOTRACE: 'echotrace', + DISCORD_TYRRRZ: 'discord-tyrrrz', + TELEGRAM_NATIVE: 'telegram-native', + TELEGRAM_NATIVE_SINGLE: 'telegram-native-single', + GOOGLE_CHAT_NATIVE: 'google-chat-native', + INSTAGRAM_NATIVE: 'instagram-native', + WHATSAPP_NATIVE: 'whatsapp-native', + QQ_NATIVE: 'qq-native', + LINE_NATIVE: 'line-native', +} as const + +export type ParserFormatId = (typeof PARSER_FORMAT_IDS)[keyof typeof PARSER_FORMAT_IDS] + +export const LEGACY_PARSER_FORMAT_ID_ALIASES = { + 'shuakami-qq-exporter': PARSER_FORMAT_IDS.QQ_SHUAKAMI, + 'shuakami-qq-exporter-chunked': PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED, + 'ycccccccy-echotrace': PARSER_FORMAT_IDS.ECHOTRACE, + 'tyrrrz-discord-exporter': PARSER_FORMAT_IDS.DISCORD_TYRRRZ, + 'whatsapp-native-txt': PARSER_FORMAT_IDS.WHATSAPP_NATIVE, + 'qq-native-txt': PARSER_FORMAT_IDS.QQ_NATIVE, + 'line-native-txt': PARSER_FORMAT_IDS.LINE_NATIVE, + 'google-chat-takeout': PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE, +} as const satisfies Record + +export function normalizeParserFormatId(formatId: string): string { + return LEGACY_PARSER_FORMAT_ID_ALIASES[formatId as keyof typeof LEGACY_PARSER_FORMAT_ID_ALIASES] ?? formatId +} + +export function isParserFormatId(formatId: string): formatId is ParserFormatId { + return Object.values(PARSER_FORMAT_IDS).some((candidate) => candidate === formatId) +} diff --git a/packages/parser/src/formats/chatlab-jsonl.ts b/packages/parser/src/formats/chatlab-jsonl.ts index 6fc97229..1af7442a 100644 --- a/packages/parser/src/formats/chatlab-jsonl.ts +++ b/packages/parser/src/formats/chatlab-jsonl.ts @@ -19,6 +19,7 @@ import * as fs from 'fs' import * as readline from 'readline' import * as path from 'path' import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -111,7 +112,7 @@ function parseLine(line: string): JsonlLine | null { // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'chatlab-jsonl', + id: PARSER_FORMAT_IDS.CHATLAB_JSONL, name: 'ChatLab JSONL', platform: KNOWN_PLATFORMS.UNKNOWN, priority: 51, // 低优先级,让其他格式先匹配 diff --git a/packages/parser/src/formats/chatlab.ts b/packages/parser/src/formats/chatlab.ts index 52558be6..131ebe01 100644 --- a/packages/parser/src/formats/chatlab.ts +++ b/packages/parser/src/formats/chatlab.ts @@ -14,6 +14,7 @@ import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import streamChain from 'stream-chain' import { KNOWN_PLATFORMS, ChatType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -48,7 +49,7 @@ function extractNameFromFilePath(filePath: string): string { // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'chatlab', + id: PARSER_FORMAT_IDS.CHATLAB, name: 'ChatLab JSON', platform: KNOWN_PLATFORMS.UNKNOWN, // ChatLab 格式可能包含多平台数据 priority: 50, // 低优先级,让其他格式先匹配 diff --git a/packages/parser/src/formats/fixture.test.ts b/packages/parser/src/formats/fixture.test.ts index 51acbec1..c0bcffca 100644 --- a/packages/parser/src/formats/fixture.test.ts +++ b/packages/parser/src/formats/fixture.test.ts @@ -65,7 +65,7 @@ const fixtures: ParserFixture[] = [ '[图片]', '', ].join('\n'), - formatId: 'qq-native-txt', + formatId: 'qq-native', expected: { meta: { name: '测试 QQ 群', platform: KNOWN_PLATFORMS.QQ, type: ChatType.GROUP }, memberIds: ['10001', '10002'], @@ -190,7 +190,7 @@ const fixtures: ParserFixture[] = [ '2024/01/02 03:06:07 P.M. - Bob: image omitted', '', ].join('\n'), - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', expected: { meta: { name: 'Alice', platform: KNOWN_PLATFORMS.WHATSAPP, type: ChatType.PRIVATE }, memberIds: ['Alice', 'Bob'], @@ -224,7 +224,7 @@ const fixtures: ParserFixture[] = [ '[2026/7/9 02:28:37] Bob:Hi without sender-space', '', ].join('\n'), - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', expected: { meta: { name: 'Alice', platform: KNOWN_PLATFORMS.WHATSAPP, type: ChatType.PRIVATE }, memberIds: ['Alice', 'Bob'], @@ -261,7 +261,7 @@ const fixtures: ParserFixture[] = [ '03:05\tBob\t[Sticker]', '', ].join('\n'), - formatId: 'line-native-txt', + formatId: 'line-native', expected: { meta: { name: 'Project Team', platform: KNOWN_PLATFORMS.LINE, type: ChatType.GROUP }, memberIds: ['Alice', 'Bob'], diff --git a/packages/parser/src/formats/google-chat-takeout.test.ts b/packages/parser/src/formats/google-chat-takeout.test.ts index f1a7744f..354aff94 100644 --- a/packages/parser/src/formats/google-chat-takeout.test.ts +++ b/packages/parser/src/formats/google-chat-takeout.test.ts @@ -105,7 +105,7 @@ async function parseGoogleChatFixture(options: { 'utf8' ) - assert.equal(detectFormat(manifestPath)?.id, 'google-chat-takeout') + assert.equal(detectFormat(manifestPath)?.id, 'google-chat-native') return await parseFileSync(manifestPath) } finally { rmSync(dir, { recursive: true, force: true }) diff --git a/packages/parser/src/formats/google-chat-takeout.ts b/packages/parser/src/formats/google-chat-takeout.ts index f0b5c9aa..28d060bb 100644 --- a/packages/parser/src/formats/google-chat-takeout.ts +++ b/packages/parser/src/formats/google-chat-takeout.ts @@ -12,6 +12,7 @@ import streamJson from 'stream-json' import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -90,7 +91,7 @@ const ENGLISH_MONTHS = new Map([ ]) export const feature: FormatFeature = { - id: 'google-chat-takeout', + id: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE, name: 'Google Chat Takeout', platform: KNOWN_PLATFORMS.GOOGLE_CHAT, priority: 24, diff --git a/packages/parser/src/formats/instagram-native.ts b/packages/parser/src/formats/instagram-native.ts index fb098ce9..99501a47 100644 --- a/packages/parser/src/formats/instagram-native.ts +++ b/packages/parser/src/formats/instagram-native.ts @@ -18,6 +18,7 @@ import * as fs from 'fs' import * as path from 'path' import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -33,7 +34,7 @@ import { getFileSize, createProgress } from '../utils' // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'instagram-native', + id: PARSER_FORMAT_IDS.INSTAGRAM_NATIVE, name: 'Instagram 官方导出', platform: KNOWN_PLATFORMS.INSTAGRAM, priority: 25, diff --git a/packages/parser/src/formats/line-native-txt.ts b/packages/parser/src/formats/line-native-txt.ts index a2580b34..ca40c76b 100644 --- a/packages/parser/src/formats/line-native-txt.ts +++ b/packages/parser/src/formats/line-native-txt.ts @@ -2,11 +2,12 @@ import * as fs from 'fs' import { KNOWN_PLATFORMS } from '@openchatlab/shared-types' import { LINE_FILENAME_SIGNATURES, LINE_HEAD_SIGNATURES, parseLineText } from '../browser/line' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, ParseEvent, ParseOptions, Parser } from '../types' import { createProgress, getFileSize } from '../utils' export const feature: FormatFeature = { - id: 'line-native-txt', + id: PARSER_FORMAT_IDS.LINE_NATIVE, name: 'LINE 官方导出 TXT', platform: KNOWN_PLATFORMS.LINE, priority: 35, diff --git a/packages/parser/src/formats/qq-native-txt.ts b/packages/parser/src/formats/qq-native-txt.ts index f8a5089a..ce707546 100644 --- a/packages/parser/src/formats/qq-native-txt.ts +++ b/packages/parser/src/formats/qq-native-txt.ts @@ -3,11 +3,12 @@ import * as readline from 'readline' import { KNOWN_PLATFORMS } from '@openchatlab/shared-types' import { QQ_HEAD_SIGNATURES, QqTextAccumulator } from '../browser/qq' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, ParseEvent, ParseOptions, Parser } from '../types' import { createProgress, getFileSize } from '../utils' export const feature: FormatFeature = { - id: 'qq-native-txt', + id: PARSER_FORMAT_IDS.QQ_NATIVE, name: 'QQ 官方导出 (TXT)', platform: KNOWN_PLATFORMS.QQ, priority: 30, diff --git a/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts b/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts index 6de679d4..ef37f268 100644 --- a/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts +++ b/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts @@ -61,7 +61,7 @@ describe('shuakami-qq-exporter chunked parser', () => { .join('\n') ) - assert.equal(detectFormat(manifestPath)?.id, 'shuakami-qq-exporter-chunked') + assert.equal(detectFormat(manifestPath)?.id, 'qq-shuakami-chunked') assert.equal(findEntryFileInDirectory(root), manifestPath) const result = await parseFileSync(manifestPath) diff --git a/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts b/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts index 7dcc9cb4..fd8f1fb7 100644 --- a/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts +++ b/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts @@ -27,6 +27,7 @@ import * as fs from 'fs' import * as path from 'path' import * as readline from 'readline' import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -42,7 +43,7 @@ import { createProgress, parseTimestamp, isValidYear } from '../utils' // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'shuakami-qq-exporter-chunked', + id: PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED, name: 'shuakami/qq-chat-exporter (chunked)', platform: KNOWN_PLATFORMS.QQ, priority: 5, // 比单文件版本优先级更高 diff --git a/packages/parser/src/formats/shuakami-qq-exporter.test.ts b/packages/parser/src/formats/shuakami-qq-exporter.test.ts index 581b73e9..b1c6df0f 100644 --- a/packages/parser/src/formats/shuakami-qq-exporter.test.ts +++ b/packages/parser/src/formats/shuakami-qq-exporter.test.ts @@ -62,7 +62,7 @@ async function parseContent(content: string, filename = 'qce-export.json'): Prom try { const filePath = join(dir, filename) writeFileSync(filePath, content, 'utf-8') - assert.equal(detectFormat(filePath)?.id, 'shuakami-qq-exporter') + assert.equal(detectFormat(filePath)?.id, 'qq-shuakami') return await parseFileSync(filePath) } finally { rmSync(dir, { recursive: true, force: true }) diff --git a/packages/parser/src/formats/shuakami-qq-exporter.ts b/packages/parser/src/formats/shuakami-qq-exporter.ts index 08bafc99..d1bb6f79 100644 --- a/packages/parser/src/formats/shuakami-qq-exporter.ts +++ b/packages/parser/src/formats/shuakami-qq-exporter.ts @@ -24,6 +24,7 @@ import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import streamChain from 'stream-chain' import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -44,7 +45,7 @@ const { chain } = streamChain // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'shuakami-qq-exporter', + id: PARSER_FORMAT_IDS.QQ_SHUAKAMI, name: 'shuakami/qq-chat-exporter', platform: KNOWN_PLATFORMS.QQ, priority: 10, diff --git a/packages/parser/src/formats/telegram-native-single.ts b/packages/parser/src/formats/telegram-native-single.ts index 46b44015..e7fe71b8 100644 --- a/packages/parser/src/formats/telegram-native-single.ts +++ b/packages/parser/src/formats/telegram-native-single.ts @@ -4,6 +4,7 @@ import streamJson from 'stream-json' import { ChatType, KNOWN_PLATFORMS } from '@openchatlab/shared-types' import { TELEGRAM_SINGLE_CHAT_TYPE_PATTERN, TELEGRAM_SINGLE_HEAD_SIGNATURES } from '../browser/telegram' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -26,7 +27,7 @@ interface JsonToken { } export const feature: FormatFeature = { - id: 'telegram-native-single', + id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE, name: 'Telegram 单聊天导出 (JSON)', platform: KNOWN_PLATFORMS.TELEGRAM, priority: 23, diff --git a/packages/parser/src/formats/telegram-native.ts b/packages/parser/src/formats/telegram-native.ts index 327d7ada..be5e60ff 100644 --- a/packages/parser/src/formats/telegram-native.ts +++ b/packages/parser/src/formats/telegram-native.ts @@ -22,6 +22,7 @@ import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import { KNOWN_PLATFORMS, ChatType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -60,7 +61,7 @@ export interface TelegramChatInfo { // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'telegram-native', + id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE, name: 'Telegram 官方导出 (JSON)', platform: KNOWN_PLATFORMS.TELEGRAM, priority: 22, diff --git a/packages/parser/src/formats/tyrrrz-discord-exporter.ts b/packages/parser/src/formats/tyrrrz-discord-exporter.ts index a0fee082..9fb9f21e 100644 --- a/packages/parser/src/formats/tyrrrz-discord-exporter.ts +++ b/packages/parser/src/formats/tyrrrz-discord-exporter.ts @@ -14,6 +14,7 @@ import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import streamChain from 'stream-chain' import { KNOWN_PLATFORMS, ChatType, MessageType, type MemberRole } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -230,7 +231,7 @@ function formatSticker(sticker: DiscordSticker): string { // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'tyrrrz-discord-exporter', + id: PARSER_FORMAT_IDS.DISCORD_TYRRRZ, name: 'Tyrrrz/DiscordChatExporter', platform: KNOWN_PLATFORMS.DISCORD, priority: 20, // Discord 格式优先级 diff --git a/packages/parser/src/formats/weflow.ts b/packages/parser/src/formats/weflow.ts index a724a69e..86d126e6 100644 --- a/packages/parser/src/formats/weflow.ts +++ b/packages/parser/src/formats/weflow.ts @@ -22,6 +22,7 @@ import pickModule from 'stream-json/filters/Pick.js' import streamValuesModule from 'stream-json/streamers/StreamValues.js' import streamChain from 'stream-chain' import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types' +import { PARSER_FORMAT_IDS } from '../format-ids' import type { FormatFeature, FormatModule, @@ -53,7 +54,7 @@ function extractNameFromFilePath(filePath: string): string { // ==================== 特征定义 ==================== export const feature: FormatFeature = { - id: 'weflow', + id: PARSER_FORMAT_IDS.WEFLOW, name: 'WeFlow 导出', platform: KNOWN_PLATFORMS.WECHAT, priority: 15, @@ -608,7 +609,7 @@ async function* parseWeFlow(options: ParseOptions): AsyncGenerator f.id === formatId) ?? null + return sniffer.getFeatureById(formatId) } /** diff --git a/packages/parser/src/native/chatlab-native.ts b/packages/parser/src/native/chatlab-native.ts index 18dbfee7..2788bfb4 100644 --- a/packages/parser/src/native/chatlab-native.ts +++ b/packages/parser/src/native/chatlab-native.ts @@ -51,7 +51,7 @@ function toParsedMember(member: NativeMember, fromHead: boolean): ParsedMember { } const chatlabAdapter: NativeFormatAdapter = { - formatId: 'chatlab', + kernelId: 'chatlab', label: 'ChatLab JSON', mapMeta(metaJson: unknown): ParsedMeta { diff --git a/packages/parser/src/native/create-native-parser.test.ts b/packages/parser/src/native/create-native-parser.test.ts index 1a64ce63..502e9111 100644 --- a/packages/parser/src/native/create-native-parser.test.ts +++ b/packages/parser/src/native/create-native-parser.test.ts @@ -19,7 +19,7 @@ test('falls back before data events when the native parser constructor rejects t const { createNativeFirstParser } = await import('./create-native-parser') const parser = createNativeFirstParser( { - formatId: 'shuakami-qq-exporter', + kernelId: 'shuakami-qq-exporter', label: 'shuakami/qq-chat-exporter', mapMeta() { throw new Error('native mapper must not run') diff --git a/packages/parser/src/native/create-native-parser.ts b/packages/parser/src/native/create-native-parser.ts index 07dbf473..c830b005 100644 --- a/packages/parser/src/native/create-native-parser.ts +++ b/packages/parser/src/native/create-native-parser.ts @@ -25,7 +25,7 @@ export type ParseGenerator = (options: ParseOptions) => AsyncGenerator { class OlderNativeParser { constructor(formatId: string) { - if (formatId !== 'chatlab') throw new Error(`unsupported format: ${formatId}`) + if (formatId !== 'chatlab' && formatId !== 'shuakami-qq-exporter') { + throw new Error(`unsupported format: ${formatId}`) + } } } @@ -14,7 +16,9 @@ test('probes kernel ids so an older loadable native binary is not treated as for const { isNativeFormatAvailable } = await import('./loader') assert.equal(isNativeFormatAvailable('chatlab'), true) - assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false) + assert.equal(isNativeFormatAvailable('qq-shuakami'), true) + assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), true) + assert.equal(isNativeFormatAvailable('discord-tyrrrz'), false) const saved = process.env.CHATLAB_DISABLE_NATIVE_PERF try { diff --git a/packages/parser/src/native/loader.test.ts b/packages/parser/src/native/loader.test.ts index 7bd647ed..69ace25c 100644 --- a/packages/parser/src/native/loader.test.ts +++ b/packages/parser/src/native/loader.test.ts @@ -16,5 +16,5 @@ test('treats a loadable native module without NativeParser as unavailable', asyn disabled: false, error: 'Native parser module missing NativeParser export', }) - assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false) + assert.equal(isNativeFormatAvailable('qq-shuakami'), false) }) diff --git a/packages/parser/src/native/loader.ts b/packages/parser/src/native/loader.ts index 021baac5..f3e35110 100644 --- a/packages/parser/src/native/loader.ts +++ b/packages/parser/src/native/loader.ts @@ -10,6 +10,7 @@ import { createRequire } from 'node:module' import type { NativeParser } from '@openchatlab/parser-native' +import { PARSER_FORMAT_IDS, normalizeParserFormatId } from '../format-ids' export interface NativeParserModule { NativeParser: typeof NativeParser @@ -81,9 +82,16 @@ export function isNativeFormatAvailable(formatId: string): boolean { if (!module) return false try { - new module.NativeParser(formatId, '') + new module.NativeParser(resolveNativeKernelId(formatId), '') return true } catch { return false } } + +function resolveNativeKernelId(formatId: string): string { + const canonicalId = normalizeParserFormatId(formatId) + if (canonicalId === PARSER_FORMAT_IDS.QQ_SHUAKAMI) return 'shuakami-qq-exporter' + if (canonicalId === PARSER_FORMAT_IDS.ECHOTRACE) return 'weflow' + return canonicalId +} diff --git a/packages/parser/src/native/shuakami-qq-native.test.ts b/packages/parser/src/native/shuakami-qq-native.test.ts index 48fa9ff0..9b564939 100644 --- a/packages/parser/src/native/shuakami-qq-native.test.ts +++ b/packages/parser/src/native/shuakami-qq-native.test.ts @@ -178,7 +178,7 @@ const nativeAvailable = (() => { const saved = process.env[ENV_KEY] delete process.env[ENV_KEY] try { - return isNativeFormatAvailable('shuakami-qq-exporter') + return isNativeFormatAvailable('qq-shuakami') } finally { if (saved !== undefined) process.env[ENV_KEY] = saved } @@ -254,7 +254,7 @@ describe('shuakami/qq-chat-exporter native parser', { skip: !nativeAvailable && try { writeFileSync(filePath, makeFullFixture(), 'utf-8') process.env[ENV_KEY] = '1' - assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false) + assert.equal(isNativeFormatAvailable('qq-shuakami'), false) const result = await collect(parseShuakamiQqV4Accelerated, filePath) assert.equal(result.messages.length, 24) assert.equal( diff --git a/packages/parser/src/native/shuakami-qq-native.ts b/packages/parser/src/native/shuakami-qq-native.ts index 74addd12..5794200f 100644 --- a/packages/parser/src/native/shuakami-qq-native.ts +++ b/packages/parser/src/native/shuakami-qq-native.ts @@ -15,7 +15,7 @@ interface ShuakamiQqMetaJson { } export const shuakamiQqAdapter: NativeFormatAdapter = { - formatId: 'shuakami-qq-exporter', + kernelId: 'shuakami-qq-exporter', label: 'shuakami/qq-chat-exporter', mapMeta(metaJson: unknown): ParsedMeta { diff --git a/packages/parser/src/native/weflow-native.test.ts b/packages/parser/src/native/weflow-native.test.ts index f96631b4..0d146603 100644 --- a/packages/parser/src/native/weflow-native.test.ts +++ b/packages/parser/src/native/weflow-native.test.ts @@ -240,11 +240,7 @@ describe('weflow native parser parity', { skip: !nativeAvailable() && 'native mo }, ], } - const { nativeResult, tsResult } = await parseBothWays( - 'echo.json', - JSON.stringify(echotrace), - 'ycccccccy-echotrace' - ) + const { nativeResult, tsResult } = await parseBothWays('echo.json', JSON.stringify(echotrace), 'echotrace') assertParity(nativeResult, tsResult) assert.equal(nativeResult.messages.length, 1) }) diff --git a/packages/parser/src/native/weflow-native.ts b/packages/parser/src/native/weflow-native.ts index 81503e35..75d607e6 100644 --- a/packages/parser/src/native/weflow-native.ts +++ b/packages/parser/src/native/weflow-native.ts @@ -2,7 +2,7 @@ * WeFlow adapter for the unified native-first wrapper. * * Payload shapes are identical to the pure-TS parser in formats/weflow.ts - * (verified by parity tests). Shared by the ycccccccy-echotrace format module + * (verified by parity tests). Shared by the echotrace format module * (same data structure, same Rust kernel). */ @@ -21,7 +21,7 @@ interface WeflowMetaJson { } const weflowAdapter: NativeFormatAdapter = { - formatId: 'weflow', + kernelId: 'weflow', label: 'WeFlow export', mapMeta(metaJson: unknown): ParsedMeta { @@ -62,7 +62,7 @@ const weflowAdapter: NativeFormatAdapter = { /** * Wrap a TS parse generator with native acceleration. Used by the WeFlow and - * ycccccccy-echotrace format modules (they share the same data structure). + * echotrace format modules (they share the same data structure). */ export function withNativeWeflow(fallback: ParseGenerator): ParseGenerator { return createNativeFirstParser(weflowAdapter, fallback) diff --git a/packages/parser/src/sniffer.ts b/packages/parser/src/sniffer.ts index d7b5bb19..d84c523f 100644 --- a/packages/parser/src/sniffer.ts +++ b/packages/parser/src/sniffer.ts @@ -5,6 +5,7 @@ import * as fs from 'fs' import * as path from 'path' +import { normalizeParserFormatId } from './format-ids' import type { FormatFeature, FormatModule, Parser } from './types' /** 文件头检测大小 (64KB) - 考虑到现代聊天记录文件可能包含 base64 头像等大数据 */ @@ -172,10 +173,19 @@ export class FormatSniffer { * 根据格式 ID 获取解析器 */ getParserById(formatId: string): Parser | null { - const module = this.formats.find((m) => m.feature.id === formatId) + const canonicalId = normalizeParserFormatId(formatId) + const module = this.formats.find((m) => m.feature.id === canonicalId) return module?.parser || null } + /** + * Resolve canonical and released legacy format IDs to one canonical feature. + */ + getFeatureById(formatId: string): FormatFeature | null { + const canonicalId = normalizeParserFormatId(formatId) + return this.formats.find((module) => module.feature.id === canonicalId)?.feature ?? null + } + /** * 获取所有支持的格式 */ diff --git a/packages/parser/src/types.ts b/packages/parser/src/types.ts index 444c544c..df763027 100644 --- a/packages/parser/src/types.ts +++ b/packages/parser/src/types.ts @@ -4,6 +4,7 @@ */ import type { ChatPlatform, ChatType, ParsedMember, ParsedMessage } from '@openchatlab/shared-types' +import type { ParserFormatId } from './format-ids' // ==================== 标准层:统一输出结构 ==================== @@ -83,7 +84,7 @@ export interface FormatSignatures { */ export interface FormatFeature { /** 唯一标识符 */ - id: string + id: ParserFormatId /** 显示名称 */ name: string /** 平台 */ diff --git a/packages/web-runtime/scripts/bench-import.mts b/packages/web-runtime/scripts/bench-import.mts index 053d201e..d84547a1 100644 --- a/packages/web-runtime/scripts/bench-import.mts +++ b/packages/web-runtime/scripts/bench-import.mts @@ -211,7 +211,7 @@ function createWhatsAppFixture(): BenchmarkFixture { const blob = new Blob([lines.join('\n')], { type: 'text/plain' }) return { label: 'WhatsApp TXT', - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', source: blobSource('WhatsApp benchmark.txt', blob), firstContent: 'fixture-message-0', lastContent: `fixture-message-${MESSAGE_COUNT - 1}`, @@ -233,7 +233,7 @@ function createLineFixture(): BenchmarkFixture { const blob = new Blob([lines.join('\n')], { type: 'text/plain' }) return { label: 'LINE TXT', - formatId: 'line-native-txt', + formatId: 'line-native', source: blobSource('[LINE] Benchmark Team.txt', blob), firstContent: 'fixture-message-0', lastContent: `fixture-message-${MESSAGE_COUNT - 1}`, @@ -255,7 +255,7 @@ function createQqFixture(): BenchmarkFixture { const blob = new Blob([lines.join('\n')], { type: 'text/plain' }) return { label: 'QQ TXT', - formatId: 'qq-native-txt', + formatId: 'qq-native', source: blobSource('qq-benchmark.txt', blob), firstContent: 'fixture-message-0', lastContent: `fixture-message-${MESSAGE_COUNT - 1}`, diff --git a/packages/web-runtime/src/import/browser-parser.ts b/packages/web-runtime/src/import/browser-parser.ts index b7b5ba61..32a5a311 100644 --- a/packages/web-runtime/src/import/browser-parser.ts +++ b/packages/web-runtime/src/import/browser-parser.ts @@ -1,4 +1,5 @@ import { + PARSER_FORMAT_IDS, detectLineText, detectQqText, detectTelegramMultiChatJson, @@ -12,6 +13,7 @@ import { parseWhatsAppText, parseWeFlowJson, scanTelegramChatsJson, + normalizeParserFormatId, type TelegramChatInfo, } from '@openchatlab/parser/browser' @@ -28,12 +30,12 @@ import { parseWithWasm, type BrowserImportLogEvent, type BrowserWasmParserLoader export type BrowserImportFormatId = | ChatLabBrowserFormatId - | 'whatsapp-native-txt' - | 'line-native-txt' - | 'qq-native-txt' - | 'telegram-native' - | 'telegram-native-single' - | 'weflow' + | typeof PARSER_FORMAT_IDS.WHATSAPP_NATIVE + | typeof PARSER_FORMAT_IDS.LINE_NATIVE + | typeof PARSER_FORMAT_IDS.QQ_NATIVE + | typeof PARSER_FORMAT_IDS.TELEGRAM_NATIVE + | typeof PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE + | typeof PARSER_FORMAT_IDS.WEFLOW export interface ParseBrowserSourceOptions { formatId?: BrowserImportFormatId @@ -51,18 +53,35 @@ export interface BrowserImportParseResult extends Omit([ + PARSER_FORMAT_IDS.CHATLAB, + PARSER_FORMAT_IDS.CHATLAB_JSONL, + PARSER_FORMAT_IDS.WEFLOW, + PARSER_FORMAT_IDS.WHATSAPP_NATIVE, + PARSER_FORMAT_IDS.LINE_NATIVE, + PARSER_FORMAT_IDS.QQ_NATIVE, + PARSER_FORMAT_IDS.TELEGRAM_NATIVE, + PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE, +]) + +export function normalizeBrowserImportFormatId(formatId: string | undefined): BrowserImportFormatId | undefined { + if (!formatId) return undefined + const canonicalId = normalizeParserFormatId(formatId) + return BROWSER_IMPORT_FORMAT_IDS.has(canonicalId) ? (canonicalId as BrowserImportFormatId) : undefined +} + export async function detectBrowserImportFormat(source: BrowserParseSource): Promise { if (source.name.toLowerCase().endsWith('.txt')) { const head = await source.slice(0, HEAD_BYTES).text() - if (detectWhatsAppText(head, source.name)) return 'whatsapp-native-txt' - if (detectQqText(head, source.name)) return 'qq-native-txt' - return detectLineText(head, source.name) ? 'line-native-txt' : null + if (detectWhatsAppText(head, source.name)) return PARSER_FORMAT_IDS.WHATSAPP_NATIVE + if (detectQqText(head, source.name)) return PARSER_FORMAT_IDS.QQ_NATIVE + return detectLineText(head, source.name) ? PARSER_FORMAT_IDS.LINE_NATIVE : null } if (source.name.toLowerCase().endsWith('.json')) { const head = await source.slice(0, HEAD_BYTES).text() - if (detectWeFlowJson(head, source.name)) return 'weflow' - if (detectTelegramMultiChatJson(head, source.name)) return 'telegram-native' - if (detectTelegramSingleJson(head, source.name)) return 'telegram-native-single' + if (detectWeFlowJson(head, source.name)) return PARSER_FORMAT_IDS.WEFLOW + if (detectTelegramMultiChatJson(head, source.name)) return PARSER_FORMAT_IDS.TELEGRAM_NATIVE + if (detectTelegramSingleJson(head, source.name)) return PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE } return detectChatLabFormat(source) } @@ -72,7 +91,7 @@ export async function scanBrowserMultiChatSource( options: Pick = {} ): Promise { const formatId = await detectBrowserImportFormat(source) - if (formatId !== 'telegram-native') { + if (formatId !== PARSER_FORMAT_IDS.TELEGRAM_NATIVE) { throw new WebRuntimeError('NOT_MULTI_CHAT_FORMAT', 'The selected file is not a supported multi-chat export') } options.checkCancelled?.() @@ -85,7 +104,9 @@ export async function parseBrowserImportSource( source: BrowserParseSource, options: ParseBrowserSourceOptions = {} ): Promise { - const formatId = options.formatId ?? (await detectBrowserImportFormat(source)) + const formatId = options.formatId + ? normalizeBrowserImportFormatId(options.formatId) + : await detectBrowserImportFormat(source) if (!formatId) { throw new WebRuntimeError( 'UNSUPPORTED_IMPORT_FORMAT', @@ -93,7 +114,7 @@ export async function parseBrowserImportSource( ) } - if (formatId === 'chatlab' || formatId === 'weflow') { + if (formatId === PARSER_FORMAT_IDS.CHATLAB || formatId === PARSER_FORMAT_IDS.WEFLOW) { const wasmResult = await parseWithWasm(source, formatId, { checkCancelled: options.checkCancelled, onProgress: options.onProgress, @@ -103,7 +124,7 @@ export async function parseBrowserImportSource( if (wasmResult) return wasmResult } - if (formatId === 'whatsapp-native-txt') { + if (formatId === PARSER_FORMAT_IDS.WHATSAPP_NATIVE) { options.checkCancelled?.() const content = await source.text() options.checkCancelled?.() @@ -120,7 +141,7 @@ export async function parseBrowserImportSource( } } - if (formatId === 'weflow') { + if (formatId === PARSER_FORMAT_IDS.WEFLOW) { options.checkCancelled?.() const content = await source.text() options.checkCancelled?.() @@ -137,7 +158,7 @@ export async function parseBrowserImportSource( } } - if (formatId === 'line-native-txt') { + if (formatId === PARSER_FORMAT_IDS.LINE_NATIVE) { options.checkCancelled?.() const content = await source.text() options.checkCancelled?.() @@ -154,7 +175,7 @@ export async function parseBrowserImportSource( } } - if (formatId === 'qq-native-txt') { + if (formatId === PARSER_FORMAT_IDS.QQ_NATIVE) { options.checkCancelled?.() const content = await source.text() options.checkCancelled?.() @@ -171,7 +192,7 @@ export async function parseBrowserImportSource( } } - if (formatId === 'telegram-native-single') { + if (formatId === PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE) { options.checkCancelled?.() const content = await source.text() options.checkCancelled?.() @@ -188,7 +209,7 @@ export async function parseBrowserImportSource( } } - if (formatId === 'telegram-native') { + if (formatId === PARSER_FORMAT_IDS.TELEGRAM_NATIVE) { if (options.chatIndex === undefined) { throw new WebRuntimeError('MULTI_CHAT_SELECTION_REQUIRED', 'A Telegram chat index is required for import') } diff --git a/packages/web-runtime/src/import/session-runtime.test.ts b/packages/web-runtime/src/import/session-runtime.test.ts index c944ebeb..df61c917 100644 --- a/packages/web-runtime/src/import/session-runtime.test.ts +++ b/packages/web-runtime/src/import/session-runtime.test.ts @@ -307,7 +307,7 @@ describe('BrowserSessionRuntime', () => { assert.deepEqual(result, { sessionId: 'whatsapp-system-participant', - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', messageCount: 1, memberCount: 1, skippedCount: 0, @@ -351,11 +351,11 @@ describe('BrowserSessionRuntime', () => { ].join('\n') ) - assert.equal(await runtime.detectFormat(fixture), 'whatsapp-native-txt') + assert.equal(await runtime.detectFormat(fixture), 'whatsapp-native') assert.deepEqual( - runtime.getSupportedFormats().find((format) => format.id === 'whatsapp-native-txt'), + runtime.getSupportedFormats().find((format) => format.id === 'whatsapp-native'), { - id: 'whatsapp-native-txt', + id: 'whatsapp-native', name: 'WhatsApp TXT', platform: 'whatsapp', extensions: ['.txt'], @@ -366,7 +366,7 @@ describe('BrowserSessionRuntime', () => { assert.deepEqual(result, { sessionId: 'whatsapp-session', - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', messageCount: 3, memberCount: 3, skippedCount: 0, @@ -384,7 +384,7 @@ describe('BrowserSessionRuntime', () => { groupAvatar: null, ownerId: null, lastMessageTs: Math.floor(new Date('2024-01-02T03:06:00').getTime() / 1000), - formatId: 'whatsapp-native-txt', + formatId: 'whatsapp-native', }, ]) const sessionDb = database.getDatabase(sessionDatabaseFilename('whatsapp-session')) @@ -514,11 +514,11 @@ describe('BrowserSessionRuntime', () => { ].join('\n') ) - assert.equal(await runtime.detectFormat(fixture), 'line-native-txt') + assert.equal(await runtime.detectFormat(fixture), 'line-native') assert.deepEqual( - runtime.getSupportedFormats().find((format) => format.id === 'line-native-txt'), + runtime.getSupportedFormats().find((format) => format.id === 'line-native'), { - id: 'line-native-txt', + id: 'line-native', name: 'LINE TXT', platform: 'line', extensions: ['.txt'], @@ -529,7 +529,7 @@ describe('BrowserSessionRuntime', () => { assert.deepEqual(result, { sessionId: 'line-session', - formatId: 'line-native-txt', + formatId: 'line-native', messageCount: 3, memberCount: 3, skippedCount: 0, @@ -541,7 +541,7 @@ describe('BrowserSessionRuntime', () => { type: session.type, formatId: session.formatId, })), - [{ name: 'Project Team', platform: 'line', type: 'group', formatId: 'line-native-txt' }] + [{ name: 'Project Team', platform: 'line', type: 'group', formatId: 'line-native' }] ) const sessionDb = database.getDatabase(sessionDatabaseFilename('line-session')) assert.deepEqual(sessionDb?.prepare('SELECT sender_account_name, type, content FROM message ORDER BY id').all(), [ @@ -572,11 +572,11 @@ describe('BrowserSessionRuntime', () => { ].join('\n') ) - assert.equal(await runtime.detectFormat(fixture), 'qq-native-txt') + assert.equal(await runtime.detectFormat(fixture), 'qq-native') assert.deepEqual( - runtime.getSupportedFormats().find((format) => format.id === 'qq-native-txt'), + runtime.getSupportedFormats().find((format) => format.id === 'qq-native'), { - id: 'qq-native-txt', + id: 'qq-native', name: 'QQ TXT', platform: 'qq', extensions: ['.txt'], @@ -587,7 +587,7 @@ describe('BrowserSessionRuntime', () => { assert.deepEqual(result, { sessionId: 'qq-session', - formatId: 'qq-native-txt', + formatId: 'qq-native', messageCount: 2, memberCount: 2, skippedCount: 0, @@ -599,7 +599,7 @@ describe('BrowserSessionRuntime', () => { type: session.type, formatId: session.formatId, })), - [{ name: 'Project Team', platform: 'qq', type: 'group', formatId: 'qq-native-txt' }] + [{ name: 'Project Team', platform: 'qq', type: 'group', formatId: 'qq-native' }] ) const sessionDb = database.getDatabase(sessionDatabaseFilename('qq-session')) assert.deepEqual(sessionDb?.prepare('SELECT sender_account_name, type, content FROM message ORDER BY id').all(), [ diff --git a/packages/web-runtime/src/import/session-runtime.ts b/packages/web-runtime/src/import/session-runtime.ts index 72de4d29..6ef71b97 100644 --- a/packages/web-runtime/src/import/session-runtime.ts +++ b/packages/web-runtime/src/import/session-runtime.ts @@ -47,6 +47,7 @@ import { type WordFrequencyResult, type YearlyActivity, } from '@openchatlab/core' +import { PARSER_FORMAT_IDS } from '@openchatlab/parser/browser' import { WebRuntimeError } from '../runtime-error' import type { WorkspaceDatabasePort, WorkspaceDatabaseStage } from '../storage/workspace-database' import { @@ -120,18 +121,18 @@ const SUPPORTED_FORMATS: BrowserImportFormatInfo[] = [ { id: 'chatlab', name: 'ChatLab JSON', platform: 'unknown', extensions: ['.json'] }, { id: 'chatlab-jsonl', name: 'ChatLab JSONL', platform: 'unknown', extensions: ['.jsonl'] }, { id: 'weflow', name: 'WeFlow JSON', platform: 'weixin', extensions: ['.json'] }, - { id: 'whatsapp-native-txt', name: 'WhatsApp TXT', platform: 'whatsapp', extensions: ['.txt'] }, - { id: 'line-native-txt', name: 'LINE TXT', platform: 'line', extensions: ['.txt'] }, - { id: 'qq-native-txt', name: 'QQ TXT', platform: 'qq', extensions: ['.txt'] }, + { id: PARSER_FORMAT_IDS.WHATSAPP_NATIVE, name: 'WhatsApp TXT', platform: 'whatsapp', extensions: ['.txt'] }, + { id: PARSER_FORMAT_IDS.LINE_NATIVE, name: 'LINE TXT', platform: 'line', extensions: ['.txt'] }, + { id: PARSER_FORMAT_IDS.QQ_NATIVE, name: 'QQ TXT', platform: 'qq', extensions: ['.txt'] }, { - id: 'telegram-native', + id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE, name: 'Telegram JSON', platform: 'telegram', extensions: ['.json'], multiChat: true, }, { - id: 'telegram-native-single', + id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE, name: 'Telegram JSON', platform: 'telegram', extensions: ['.json'], diff --git a/packages/web-runtime/src/index.ts b/packages/web-runtime/src/index.ts index a8cc4f9b..11ebc5f3 100644 --- a/packages/web-runtime/src/index.ts +++ b/packages/web-runtime/src/index.ts @@ -14,7 +14,11 @@ export type { WebRuntimeTaskResult, WebRuntimeTaskType, } from './rpc/protocol' -export type { BrowserImportFormatId, BrowserParseSource } from './import/browser-parser' +export { + normalizeBrowserImportFormatId, + type BrowserImportFormatId, + type BrowserParseSource, +} from './import/browser-parser' export type { BrowserSessionCatalogItem } from './import/session-catalog' export type { BrowserImportFormatInfo, diff --git a/src/services/import/browser.test.ts b/src/services/import/browser.test.ts index 49fe5f12..6055786e 100644 --- a/src/services/import/browser.test.ts +++ b/src/services/import/browser.test.ts @@ -295,7 +295,7 @@ describe('BrowserImportAdapter', () => { assert.deepEqual(deletedSessionIds, ['session-1']) }) - it('forwards browser-safe format identifiers to the worker runtime', async () => { + it('normalizes released browser format aliases before forwarding them to the worker runtime', async () => { const requestedFormats: string[] = [] const rpc = { async request( @@ -347,9 +347,9 @@ describe('BrowserImportAdapter', () => { assert.equal(telegram.success, true) assert.deepEqual(requestedFormats, [ 'weflow', - 'whatsapp-native-txt', - 'line-native-txt', - 'qq-native-txt', + 'whatsapp-native', + 'line-native', + 'qq-native', 'telegram-native-single', ]) }) diff --git a/src/services/import/browser.ts b/src/services/import/browser.ts index f159f71b..3acae008 100644 --- a/src/services/import/browser.ts +++ b/src/services/import/browser.ts @@ -1,4 +1,9 @@ -import type { BrowserImportFormatId, BrowserParseSource, RpcProgressPayload } from '@openchatlab/web-runtime' +import { + normalizeBrowserImportFormatId, + type BrowserImportFormatId, + type BrowserParseSource, + type RpcProgressPayload, +} from '@openchatlab/web-runtime' import { rebaseChatLabDemoDocuments } from '@openchatlab/parser/browser' import { reportRuntimeLog } from '@/services/log-report' import type { BrowserRuntimeRpcPort } from '../browser-runtime/types' @@ -38,7 +43,7 @@ export class BrowserImportAdapter implements ImportAdapter { if (typeof file === 'string') return { success: false, error: 'File path import is not available in Web WASM' } if (this.activeImport) return { success: false, error: 'Another Web WASM import is already running' } - const formatId = normalizeFormatId(options?.formatId) + const formatId = normalizeBrowserImportFormatId(options?.formatId) if (options?.formatId && !formatId) { return { success: false, error: `Unsupported Web WASM import format: ${options.formatId}` } } @@ -281,19 +286,6 @@ function mapImportProgress(progress: RpcProgressPayload): ImportProgress { return mapped } -function normalizeFormatId(formatId: string | undefined): BrowserImportFormatId | undefined { - return formatId === 'chatlab' || - formatId === 'chatlab-jsonl' || - formatId === 'weflow' || - formatId === 'whatsapp-native-txt' || - formatId === 'line-native-txt' || - formatId === 'qq-native-txt' || - formatId === 'telegram-native' || - formatId === 'telegram-native-single' - ? formatId - : undefined -} - function unsupported(capability: string): Promise { return Promise.reject(new Error(`${capability} is not available in Web WASM`)) } diff --git a/src/services/import/importSourceAdapters.test.ts b/src/services/import/importSourceAdapters.test.ts index 866c7aed..dd1ffecb 100644 --- a/src/services/import/importSourceAdapters.test.ts +++ b/src/services/import/importSourceAdapters.test.ts @@ -134,7 +134,7 @@ describe('archive import source adapters', () => { success: true, source: { sourceId: 'source-1', - formatId: 'google-chat-takeout', + formatId: 'google-chat-native', platform: 'google-chat', chats: [], expiresAt: 123, @@ -184,7 +184,7 @@ describe('archive import source adapters', () => { success: true, source: { sourceId: 'source-2', - formatId: 'google-chat-takeout', + formatId: 'google-chat-native', platform: 'google-chat', chats: [], expiresAt: 456,