mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
refactor(parser): standardize format identifiers
This commit is contained in:
1 parent
fdd0856b5d
commit
d5f34af445
47 files changed
+251
-120
No files matched your search
@@ -12,6 +12,7 @@ import {
|
||||
importDemoSessions,
|
||||
type DatabaseManager,
|
||||
} from '@openchatlab/node-runtime'
|
||||
import { PARSER_FORMAT_IDS } from '@openchatlab/parser'
|
||||
import {
|
||||
autoImport,
|
||||
autoImportBatch,
|
||||
@@ -84,7 +85,7 @@ export function registerImportRoutes(
|
||||
options.runPreparedImport ??
|
||||
(async (manifestPath: string, onProgress: (progress: unknown) => void, sessionGapThreshold?: number) => {
|
||||
const result = await runAutoImport(manifestPath, {
|
||||
formatId: 'google-chat-takeout',
|
||||
formatId: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE,
|
||||
sessionGapThreshold,
|
||||
nativeBinding: resolveNativeBinding(),
|
||||
onProgress: onProgress as any,
|
||||
|
||||
@@ -11,6 +11,7 @@ import { appLogger } from '@openchatlab/node-runtime'
|
||||
import * as databaseCore from '../database/core'
|
||||
import * as worker from '../worker/workerManager'
|
||||
import {
|
||||
PARSER_FORMAT_IDS,
|
||||
detectFormat,
|
||||
findEntryFileInDirectory,
|
||||
getSupportedFormats,
|
||||
@@ -155,7 +156,7 @@ export function registerChatHandlers(ctx: IpcContext): void {
|
||||
worker.autoImport(
|
||||
manifestPath,
|
||||
forwardImportProgress,
|
||||
{ formatId: 'google-chat-takeout' },
|
||||
{ formatId: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE },
|
||||
undefined,
|
||||
options?.sessionGapThreshold
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ import streamChain from 'stream-chain'
|
||||
import streamJson from 'stream-json'
|
||||
import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import { PARSER_FORMAT_IDS } from '@openchatlab/parser'
|
||||
import { ArchiveImportError } from './errors'
|
||||
import type { ZipArchiveReader } from './archive-reader'
|
||||
import type { ArchiveEntrySummary, ArchiveResolver, MaterializedImport, PreparedImportChat } from './types'
|
||||
@@ -88,7 +89,7 @@ function deriveConversationName(draft: ConversationDraft, ownerEmail: string | n
|
||||
}
|
||||
|
||||
export class GoogleChatTakeoutResolver implements ArchiveResolver {
|
||||
readonly id = 'google-chat-takeout'
|
||||
readonly id = PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE
|
||||
readonly platform = 'google-chat'
|
||||
|
||||
detect(entries: ArchiveEntrySummary[]): boolean {
|
||||
|
||||
@@ -459,7 +459,7 @@ test('streamingImport builds the session index exactly once with the requested g
|
||||
test(
|
||||
'streamingImport persists shuakami/qq-chat-exporter V4 native-first output without a derived FTS table',
|
||||
{
|
||||
skip: !isNativeFormatAvailable('shuakami-qq-exporter') && 'native shuakami/qq-chat-exporter kernel not built',
|
||||
skip: !isNativeFormatAvailable('qq-shuakami') && 'native shuakami/qq-chat-exporter kernel not built',
|
||||
},
|
||||
async (t) => {
|
||||
const root = makeTempDir()
|
||||
@@ -529,7 +529,7 @@ test('streamingImport skips preprocessing when the selected format has native su
|
||||
isNativeFormatAvailable: () => true,
|
||||
}
|
||||
|
||||
const result = await streamingImport(filePath, deps, { formatId: 'shuakami-qq-exporter' }, 'native-preprocess-gate')
|
||||
const result = await streamingImport(filePath, deps, { formatId: 'qq-shuakami' }, 'native-preprocess-gate')
|
||||
|
||||
assert.equal(result.success, true)
|
||||
assert.equal(result.diagnostics?.messagesWritten, 2)
|
||||
@@ -538,7 +538,7 @@ test('streamingImport skips preprocessing when the selected format has native su
|
||||
assert.equal(fixture.getTempFilePath(), null)
|
||||
})
|
||||
|
||||
test('streamingImport preprocesses directly when native support is unavailable', async (t) => {
|
||||
test('streamingImport accepts a released format alias when native support is unavailable', async (t) => {
|
||||
const root = makeTempDir()
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
|
||||
const filePath = writeSingleFileShuakamiQqExport(root)
|
||||
@@ -586,7 +586,7 @@ test('streamingImport cleans a generated shuakami/qq-chat-exporter slim file whe
|
||||
isNativeFormatAvailable: () => false,
|
||||
},
|
||||
},
|
||||
{ formatId: 'shuakami-qq-exporter' },
|
||||
{ formatId: 'qq-shuakami' },
|
||||
'shuakami-qq-preprocess-setup-failure'
|
||||
),
|
||||
/database setup failed for test/
|
||||
|
||||
@@ -31,7 +31,7 @@ use protocol::{NativeMember, NativeParseProgress};
|
||||
// ==================== Format registry ====================
|
||||
|
||||
/// Kernel ids understood by the `NativeParser` constructor. The TS side maps
|
||||
/// sniffer format ids onto these (e.g. ycccccccy-echotrace reuses "weflow").
|
||||
/// public parser ids onto these (e.g. echotrace reuses "weflow").
|
||||
#[derive(Clone, Copy)]
|
||||
enum FormatKind {
|
||||
Weflow,
|
||||
|
||||
@@ -25,3 +25,10 @@ export {
|
||||
type WeFlowJsonParseResult,
|
||||
} from './src/browser/weflow'
|
||||
export { rebaseChatLabDemoDocuments, type RebasedChatLabDemoDocuments } from './src/chatlab-demo-timeline'
|
||||
export {
|
||||
PARSER_FORMAT_IDS,
|
||||
LEGACY_PARSER_FORMAT_ID_ALIASES,
|
||||
isParserFormatId,
|
||||
normalizeParserFormatId,
|
||||
type ParserFormatId,
|
||||
} from './src/format-ids'
|
||||
@@ -0,0 +1,27 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
|
||||
import {
|
||||
getFormatFeatureById,
|
||||
getSupportedFormats,
|
||||
LEGACY_PARSER_FORMAT_ID_ALIASES,
|
||||
normalizeParserFormatId,
|
||||
} from './index'
|
||||
|
||||
describe('parser format ID compatibility', () => {
|
||||
it('advertises canonical IDs while resolving every released legacy input alias', () => {
|
||||
const supportedIds = new Set<string>(getSupportedFormats().map((format) => format.id))
|
||||
|
||||
for (const [legacyId, canonicalId] of Object.entries(LEGACY_PARSER_FORMAT_ID_ALIASES)) {
|
||||
assert.equal(normalizeParserFormatId(legacyId), canonicalId)
|
||||
assert.equal(getFormatFeatureById(legacyId)?.id, canonicalId)
|
||||
assert.equal(supportedIds.has(canonicalId), true)
|
||||
assert.equal(supportedIds.has(legacyId), false)
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves unknown IDs unchanged so callers can reject them normally', () => {
|
||||
assert.equal(normalizeParserFormatId('future-format'), 'future-format')
|
||||
assert.equal(getFormatFeatureById('future-format'), null)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,37 @@
|
||||
export const PARSER_FORMAT_IDS = {
|
||||
CHATLAB: 'chatlab',
|
||||
CHATLAB_JSONL: 'chatlab-jsonl',
|
||||
QQ_SHUAKAMI: 'qq-shuakami',
|
||||
QQ_SHUAKAMI_CHUNKED: 'qq-shuakami-chunked',
|
||||
WEFLOW: 'weflow',
|
||||
ECHOTRACE: 'echotrace',
|
||||
DISCORD_TYRRRZ: 'discord-tyrrrz',
|
||||
TELEGRAM_NATIVE: 'telegram-native',
|
||||
TELEGRAM_NATIVE_SINGLE: 'telegram-native-single',
|
||||
GOOGLE_CHAT_NATIVE: 'google-chat-native',
|
||||
INSTAGRAM_NATIVE: 'instagram-native',
|
||||
WHATSAPP_NATIVE: 'whatsapp-native',
|
||||
QQ_NATIVE: 'qq-native',
|
||||
LINE_NATIVE: 'line-native',
|
||||
} as const
|
||||
|
||||
export type ParserFormatId = (typeof PARSER_FORMAT_IDS)[keyof typeof PARSER_FORMAT_IDS]
|
||||
|
||||
export const LEGACY_PARSER_FORMAT_ID_ALIASES = {
|
||||
'shuakami-qq-exporter': PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
||||
'shuakami-qq-exporter-chunked': PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
||||
'ycccccccy-echotrace': PARSER_FORMAT_IDS.ECHOTRACE,
|
||||
'tyrrrz-discord-exporter': PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
||||
'whatsapp-native-txt': PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
||||
'qq-native-txt': PARSER_FORMAT_IDS.QQ_NATIVE,
|
||||
'line-native-txt': PARSER_FORMAT_IDS.LINE_NATIVE,
|
||||
'google-chat-takeout': PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE,
|
||||
} as const satisfies Record<string, ParserFormatId>
|
||||
|
||||
export function normalizeParserFormatId(formatId: string): string {
|
||||
return LEGACY_PARSER_FORMAT_ID_ALIASES[formatId as keyof typeof LEGACY_PARSER_FORMAT_ID_ALIASES] ?? formatId
|
||||
}
|
||||
|
||||
export function isParserFormatId(formatId: string): formatId is ParserFormatId {
|
||||
return Object.values(PARSER_FORMAT_IDS).some((candidate) => candidate === formatId)
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import * as fs from 'fs'
|
||||
import * as readline from 'readline'
|
||||
import * as path from 'path'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -111,7 +112,7 @@ function parseLine(line: string): JsonlLine | null {
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'chatlab-jsonl',
|
||||
id: PARSER_FORMAT_IDS.CHATLAB_JSONL,
|
||||
name: 'ChatLab JSONL',
|
||||
platform: KNOWN_PLATFORMS.UNKNOWN,
|
||||
priority: 51, // 低优先级,让其他格式先匹配
|
||||
|
||||
@@ -14,6 +14,7 @@ import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import streamChain from 'stream-chain'
|
||||
import { KNOWN_PLATFORMS, ChatType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -48,7 +49,7 @@ function extractNameFromFilePath(filePath: string): string {
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'chatlab',
|
||||
id: PARSER_FORMAT_IDS.CHATLAB,
|
||||
name: 'ChatLab JSON',
|
||||
platform: KNOWN_PLATFORMS.UNKNOWN, // ChatLab 格式可能包含多平台数据
|
||||
priority: 50, // 低优先级,让其他格式先匹配
|
||||
|
||||
@@ -65,7 +65,7 @@ const fixtures: ParserFixture[] = [
|
||||
'[图片]',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'qq-native-txt',
|
||||
formatId: 'qq-native',
|
||||
expected: {
|
||||
meta: { name: '测试 QQ 群', platform: KNOWN_PLATFORMS.QQ, type: ChatType.GROUP },
|
||||
memberIds: ['10001', '10002'],
|
||||
@@ -190,7 +190,7 @@ const fixtures: ParserFixture[] = [
|
||||
'2024/01/02 03:06:07 P.M. - Bob: image omitted',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
expected: {
|
||||
meta: { name: 'Alice', platform: KNOWN_PLATFORMS.WHATSAPP, type: ChatType.PRIVATE },
|
||||
memberIds: ['Alice', 'Bob'],
|
||||
@@ -224,7 +224,7 @@ const fixtures: ParserFixture[] = [
|
||||
'[2026/7/9 02:28:37] Bob:Hi without sender-space',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
expected: {
|
||||
meta: { name: 'Alice', platform: KNOWN_PLATFORMS.WHATSAPP, type: ChatType.PRIVATE },
|
||||
memberIds: ['Alice', 'Bob'],
|
||||
@@ -261,7 +261,7 @@ const fixtures: ParserFixture[] = [
|
||||
'03:05\tBob\t[Sticker]',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'line-native-txt',
|
||||
formatId: 'line-native',
|
||||
expected: {
|
||||
meta: { name: 'Project Team', platform: KNOWN_PLATFORMS.LINE, type: ChatType.GROUP },
|
||||
memberIds: ['Alice', 'Bob'],
|
||||
|
||||
@@ -105,7 +105,7 @@ async function parseGoogleChatFixture(options: {
|
||||
'utf8'
|
||||
)
|
||||
|
||||
assert.equal(detectFormat(manifestPath)?.id, 'google-chat-takeout')
|
||||
assert.equal(detectFormat(manifestPath)?.id, 'google-chat-native')
|
||||
return await parseFileSync(manifestPath)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
|
||||
@@ -12,6 +12,7 @@ import streamJson from 'stream-json'
|
||||
import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -90,7 +91,7 @@ const ENGLISH_MONTHS = new Map([
|
||||
])
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'google-chat-takeout',
|
||||
id: PARSER_FORMAT_IDS.GOOGLE_CHAT_NATIVE,
|
||||
name: 'Google Chat Takeout',
|
||||
platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
|
||||
priority: 24,
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -33,7 +34,7 @@ import { getFileSize, createProgress } from '../utils'
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'instagram-native',
|
||||
id: PARSER_FORMAT_IDS.INSTAGRAM_NATIVE,
|
||||
name: 'Instagram 官方导出',
|
||||
platform: KNOWN_PLATFORMS.INSTAGRAM,
|
||||
priority: 25,
|
||||
|
||||
@@ -2,11 +2,12 @@ import * as fs from 'fs'
|
||||
import { KNOWN_PLATFORMS } from '@openchatlab/shared-types'
|
||||
|
||||
import { LINE_FILENAME_SIGNATURES, LINE_HEAD_SIGNATURES, parseLineText } from '../browser/line'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type { FormatFeature, FormatModule, ParseEvent, ParseOptions, Parser } from '../types'
|
||||
import { createProgress, getFileSize } from '../utils'
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'line-native-txt',
|
||||
id: PARSER_FORMAT_IDS.LINE_NATIVE,
|
||||
name: 'LINE 官方导出 TXT',
|
||||
platform: KNOWN_PLATFORMS.LINE,
|
||||
priority: 35,
|
||||
|
||||
@@ -3,11 +3,12 @@ import * as readline from 'readline'
|
||||
import { KNOWN_PLATFORMS } from '@openchatlab/shared-types'
|
||||
|
||||
import { QQ_HEAD_SIGNATURES, QqTextAccumulator } from '../browser/qq'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type { FormatFeature, FormatModule, ParseEvent, ParseOptions, Parser } from '../types'
|
||||
import { createProgress, getFileSize } from '../utils'
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'qq-native-txt',
|
||||
id: PARSER_FORMAT_IDS.QQ_NATIVE,
|
||||
name: 'QQ 官方导出 (TXT)',
|
||||
platform: KNOWN_PLATFORMS.QQ,
|
||||
priority: 30,
|
||||
|
||||
@@ -61,7 +61,7 @@ describe('shuakami-qq-exporter chunked parser', () => {
|
||||
.join('\n')
|
||||
)
|
||||
|
||||
assert.equal(detectFormat(manifestPath)?.id, 'shuakami-qq-exporter-chunked')
|
||||
assert.equal(detectFormat(manifestPath)?.id, 'qq-shuakami-chunked')
|
||||
assert.equal(findEntryFileInDirectory(root), manifestPath)
|
||||
|
||||
const result = await parseFileSync(manifestPath)
|
||||
|
||||
@@ -27,6 +27,7 @@ import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as readline from 'readline'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -42,7 +43,7 @@ import { createProgress, parseTimestamp, isValidYear } from '../utils'
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'shuakami-qq-exporter-chunked',
|
||||
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI_CHUNKED,
|
||||
name: 'shuakami/qq-chat-exporter (chunked)',
|
||||
platform: KNOWN_PLATFORMS.QQ,
|
||||
priority: 5, // 比单文件版本优先级更高
|
||||
|
||||
@@ -62,7 +62,7 @@ async function parseContent(content: string, filename = 'qce-export.json'): Prom
|
||||
try {
|
||||
const filePath = join(dir, filename)
|
||||
writeFileSync(filePath, content, 'utf-8')
|
||||
assert.equal(detectFormat(filePath)?.id, 'shuakami-qq-exporter')
|
||||
assert.equal(detectFormat(filePath)?.id, 'qq-shuakami')
|
||||
return await parseFileSync(filePath)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
|
||||
@@ -24,6 +24,7 @@ import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import streamChain from 'stream-chain'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -44,7 +45,7 @@ const { chain } = streamChain
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'shuakami-qq-exporter',
|
||||
id: PARSER_FORMAT_IDS.QQ_SHUAKAMI,
|
||||
name: 'shuakami/qq-chat-exporter',
|
||||
platform: KNOWN_PLATFORMS.QQ,
|
||||
priority: 10,
|
||||
|
||||
@@ -4,6 +4,7 @@ import streamJson from 'stream-json'
|
||||
import { ChatType, KNOWN_PLATFORMS } from '@openchatlab/shared-types'
|
||||
|
||||
import { TELEGRAM_SINGLE_CHAT_TYPE_PATTERN, TELEGRAM_SINGLE_HEAD_SIGNATURES } from '../browser/telegram'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -26,7 +27,7 @@ interface JsonToken {
|
||||
}
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'telegram-native-single',
|
||||
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE,
|
||||
name: 'Telegram 单聊天导出 (JSON)',
|
||||
platform: KNOWN_PLATFORMS.TELEGRAM,
|
||||
priority: 23,
|
||||
|
||||
@@ -22,6 +22,7 @@ import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
|
||||
import { KNOWN_PLATFORMS, ChatType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -60,7 +61,7 @@ export interface TelegramChatInfo {
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'telegram-native',
|
||||
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE,
|
||||
name: 'Telegram 官方导出 (JSON)',
|
||||
platform: KNOWN_PLATFORMS.TELEGRAM,
|
||||
priority: 22,
|
||||
|
||||
@@ -14,6 +14,7 @@ import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import streamChain from 'stream-chain'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType, type MemberRole } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -230,7 +231,7 @@ function formatSticker(sticker: DiscordSticker): string {
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'tyrrrz-discord-exporter',
|
||||
id: PARSER_FORMAT_IDS.DISCORD_TYRRRZ,
|
||||
name: 'Tyrrrz/DiscordChatExporter',
|
||||
platform: KNOWN_PLATFORMS.DISCORD,
|
||||
priority: 20, // Discord 格式优先级
|
||||
|
||||
@@ -22,6 +22,7 @@ import pickModule from 'stream-json/filters/Pick.js'
|
||||
import streamValuesModule from 'stream-json/streamers/StreamValues.js'
|
||||
import streamChain from 'stream-chain'
|
||||
import { KNOWN_PLATFORMS, ChatType, MessageType } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type {
|
||||
FormatFeature,
|
||||
FormatModule,
|
||||
@@ -53,7 +54,7 @@ function extractNameFromFilePath(filePath: string): string {
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'weflow',
|
||||
id: PARSER_FORMAT_IDS.WEFLOW,
|
||||
name: 'WeFlow 导出',
|
||||
platform: KNOWN_PLATFORMS.WECHAT,
|
||||
priority: 15,
|
||||
@@ -608,7 +609,7 @@ async function* parseWeFlow(options: ParseOptions): AsyncGenerator<ParseEvent, v
|
||||
|
||||
import { withNativeWeflow } from '../native/weflow-native'
|
||||
|
||||
// 导出解析函数供其他格式复用(如 ycccccccy-echotrace)
|
||||
// 导出解析函数供其他格式复用(如 echotrace)
|
||||
// parseWeFlowAccelerated:优先走 Rust 内核,native 不可用/失败时自动回退本文件的 TS 实现
|
||||
export { parseWeFlow }
|
||||
export const parseWeFlowAccelerated = withNativeWeflow(parseWeFlow)
|
||||
|
||||
@@ -3,11 +3,12 @@ import * as readline from 'readline'
|
||||
import { KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
|
||||
|
||||
import { WHATSAPP_FILENAME_SIGNATURES, WHATSAPP_HEAD_SIGNATURES, WhatsAppTextAccumulator } from '../browser/whatsapp'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type { FormatFeature, FormatModule, ParseEvent, ParseOptions, Parser } from '../types'
|
||||
import { createProgress, getFileSize } from '../utils'
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'whatsapp-native-txt',
|
||||
id: PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
||||
name: 'WhatsApp 官方导出 (TXT)',
|
||||
platform: KNOWN_PLATFORMS.WHATSAPP,
|
||||
priority: 25,
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
*/
|
||||
|
||||
import { KNOWN_PLATFORMS } from '@openchatlab/shared-types'
|
||||
import { PARSER_FORMAT_IDS } from '../format-ids'
|
||||
import type { FormatFeature, FormatModule, Parser } from '../types'
|
||||
import { parseWeFlowAccelerated } from './weflow'
|
||||
import { weflowPreprocessor } from './weflow-preprocessor'
|
||||
@@ -20,7 +21,7 @@ import { weflowPreprocessor } from './weflow-preprocessor'
|
||||
// ==================== 特征定义 ====================
|
||||
|
||||
export const feature: FormatFeature = {
|
||||
id: 'ycccccccy-echotrace',
|
||||
id: PARSER_FORMAT_IDS.ECHOTRACE,
|
||||
name: 'ycccccccy/echotrace 导出',
|
||||
platform: KNOWN_PLATFORMS.WECHAT,
|
||||
priority: 16, // 比 WeFlow (15) 略低,优先检测 WeFlow
|
||||
|
||||
@@ -31,6 +31,13 @@ export {
|
||||
type ChatLabValidationStats,
|
||||
} from './chatlab-validator'
|
||||
export { rebaseChatLabDemoDocuments, type RebasedChatLabDemoDocuments } from './chatlab-demo-timeline'
|
||||
export {
|
||||
PARSER_FORMAT_IDS,
|
||||
LEGACY_PARSER_FORMAT_ID_ALIASES,
|
||||
isParserFormatId,
|
||||
normalizeParserFormatId,
|
||||
type ParserFormatId,
|
||||
} from './format-ids'
|
||||
|
||||
// ==================== 全局嗅探器实例 ====================
|
||||
|
||||
@@ -79,8 +86,7 @@ export function getSupportedFormats(): FormatFeature[] {
|
||||
* 用于手动指定格式时跳过嗅探
|
||||
*/
|
||||
export function getFormatFeatureById(formatId: string): FormatFeature | null {
|
||||
const all = sniffer.getSupportedFormats()
|
||||
return all.find((f) => f.id === formatId) ?? null
|
||||
return sniffer.getFeatureById(formatId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -51,7 +51,7 @@ function toParsedMember(member: NativeMember, fromHead: boolean): ParsedMember {
|
||||
}
|
||||
|
||||
const chatlabAdapter: NativeFormatAdapter = {
|
||||
formatId: 'chatlab',
|
||||
kernelId: 'chatlab',
|
||||
label: 'ChatLab JSON',
|
||||
|
||||
mapMeta(metaJson: unknown): ParsedMeta {
|
||||
|
||||
@@ -19,7 +19,7 @@ test('falls back before data events when the native parser constructor rejects t
|
||||
const { createNativeFirstParser } = await import('./create-native-parser')
|
||||
const parser = createNativeFirstParser(
|
||||
{
|
||||
formatId: 'shuakami-qq-exporter',
|
||||
kernelId: 'shuakami-qq-exporter',
|
||||
label: 'shuakami/qq-chat-exporter',
|
||||
mapMeta() {
|
||||
throw new Error('native mapper must not run')
|
||||
|
||||
@@ -25,7 +25,7 @@ export type ParseGenerator = (options: ParseOptions) => AsyncGenerator<ParseEven
|
||||
|
||||
export interface NativeFormatAdapter {
|
||||
/** Kernel id understood by the Rust NativeParser constructor. */
|
||||
formatId: string
|
||||
kernelId: string
|
||||
/** Human-readable format label used in import logs. */
|
||||
label: string
|
||||
/** Map the kernel's meta JSON (parsed) to the ParsedMeta event payload. */
|
||||
@@ -127,7 +127,7 @@ export function createNativeFirstParser(
|
||||
let parser: NativeParser
|
||||
try {
|
||||
parser = new native.NativeParser(
|
||||
adapter.formatId,
|
||||
adapter.kernelId,
|
||||
options.filePath,
|
||||
options.formatOptions ? JSON.stringify(options.formatOptions) : undefined
|
||||
)
|
||||
|
||||
@@ -4,7 +4,9 @@ import test from 'node:test'
|
||||
test('probes kernel ids so an older loadable native binary is not treated as format-capable', async (t) => {
|
||||
class OlderNativeParser {
|
||||
constructor(formatId: string) {
|
||||
if (formatId !== 'chatlab') throw new Error(`unsupported format: ${formatId}`)
|
||||
if (formatId !== 'chatlab' && formatId !== 'shuakami-qq-exporter') {
|
||||
throw new Error(`unsupported format: ${formatId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +16,9 @@ test('probes kernel ids so an older loadable native binary is not treated as for
|
||||
|
||||
const { isNativeFormatAvailable } = await import('./loader')
|
||||
assert.equal(isNativeFormatAvailable('chatlab'), true)
|
||||
assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false)
|
||||
assert.equal(isNativeFormatAvailable('qq-shuakami'), true)
|
||||
assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), true)
|
||||
assert.equal(isNativeFormatAvailable('discord-tyrrrz'), false)
|
||||
|
||||
const saved = process.env.CHATLAB_DISABLE_NATIVE_PERF
|
||||
try {
|
||||
|
||||
@@ -16,5 +16,5 @@ test('treats a loadable native module without NativeParser as unavailable', asyn
|
||||
disabled: false,
|
||||
error: 'Native parser module missing NativeParser export',
|
||||
})
|
||||
assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false)
|
||||
assert.equal(isNativeFormatAvailable('qq-shuakami'), false)
|
||||
})
|
||||
@@ -10,6 +10,7 @@
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
import type { NativeParser } from '@openchatlab/parser-native'
|
||||
import { PARSER_FORMAT_IDS, normalizeParserFormatId } from '../format-ids'
|
||||
|
||||
export interface NativeParserModule {
|
||||
NativeParser: typeof NativeParser
|
||||
@@ -81,9 +82,16 @@ export function isNativeFormatAvailable(formatId: string): boolean {
|
||||
if (!module) return false
|
||||
|
||||
try {
|
||||
new module.NativeParser(formatId, '')
|
||||
new module.NativeParser(resolveNativeKernelId(formatId), '')
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function resolveNativeKernelId(formatId: string): string {
|
||||
const canonicalId = normalizeParserFormatId(formatId)
|
||||
if (canonicalId === PARSER_FORMAT_IDS.QQ_SHUAKAMI) return 'shuakami-qq-exporter'
|
||||
if (canonicalId === PARSER_FORMAT_IDS.ECHOTRACE) return 'weflow'
|
||||
return canonicalId
|
||||
}
|
||||
@@ -178,7 +178,7 @@ const nativeAvailable = (() => {
|
||||
const saved = process.env[ENV_KEY]
|
||||
delete process.env[ENV_KEY]
|
||||
try {
|
||||
return isNativeFormatAvailable('shuakami-qq-exporter')
|
||||
return isNativeFormatAvailable('qq-shuakami')
|
||||
} finally {
|
||||
if (saved !== undefined) process.env[ENV_KEY] = saved
|
||||
}
|
||||
@@ -254,7 +254,7 @@ describe('shuakami/qq-chat-exporter native parser', { skip: !nativeAvailable &&
|
||||
try {
|
||||
writeFileSync(filePath, makeFullFixture(), 'utf-8')
|
||||
process.env[ENV_KEY] = '1'
|
||||
assert.equal(isNativeFormatAvailable('shuakami-qq-exporter'), false)
|
||||
assert.equal(isNativeFormatAvailable('qq-shuakami'), false)
|
||||
const result = await collect(parseShuakamiQqV4Accelerated, filePath)
|
||||
assert.equal(result.messages.length, 24)
|
||||
assert.equal(
|
||||
|
||||
@@ -15,7 +15,7 @@ interface ShuakamiQqMetaJson {
|
||||
}
|
||||
|
||||
export const shuakamiQqAdapter: NativeFormatAdapter = {
|
||||
formatId: 'shuakami-qq-exporter',
|
||||
kernelId: 'shuakami-qq-exporter',
|
||||
label: 'shuakami/qq-chat-exporter',
|
||||
|
||||
mapMeta(metaJson: unknown): ParsedMeta {
|
||||
|
||||
@@ -240,11 +240,7 @@ describe('weflow native parser parity', { skip: !nativeAvailable() && 'native mo
|
||||
},
|
||||
],
|
||||
}
|
||||
const { nativeResult, tsResult } = await parseBothWays(
|
||||
'echo.json',
|
||||
JSON.stringify(echotrace),
|
||||
'ycccccccy-echotrace'
|
||||
)
|
||||
const { nativeResult, tsResult } = await parseBothWays('echo.json', JSON.stringify(echotrace), 'echotrace')
|
||||
assertParity(nativeResult, tsResult)
|
||||
assert.equal(nativeResult.messages.length, 1)
|
||||
})
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* WeFlow adapter for the unified native-first wrapper.
|
||||
*
|
||||
* Payload shapes are identical to the pure-TS parser in formats/weflow.ts
|
||||
* (verified by parity tests). Shared by the ycccccccy-echotrace format module
|
||||
* (verified by parity tests). Shared by the echotrace format module
|
||||
* (same data structure, same Rust kernel).
|
||||
*/
|
||||
|
||||
@@ -21,7 +21,7 @@ interface WeflowMetaJson {
|
||||
}
|
||||
|
||||
const weflowAdapter: NativeFormatAdapter = {
|
||||
formatId: 'weflow',
|
||||
kernelId: 'weflow',
|
||||
label: 'WeFlow export',
|
||||
|
||||
mapMeta(metaJson: unknown): ParsedMeta {
|
||||
@@ -62,7 +62,7 @@ const weflowAdapter: NativeFormatAdapter = {
|
||||
|
||||
/**
|
||||
* Wrap a TS parse generator with native acceleration. Used by the WeFlow and
|
||||
* ycccccccy-echotrace format modules (they share the same data structure).
|
||||
* echotrace format modules (they share the same data structure).
|
||||
*/
|
||||
export function withNativeWeflow(fallback: ParseGenerator): ParseGenerator {
|
||||
return createNativeFirstParser(weflowAdapter, fallback)
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { normalizeParserFormatId } from './format-ids'
|
||||
import type { FormatFeature, FormatModule, Parser } from './types'
|
||||
|
||||
/** 文件头检测大小 (64KB) - 考虑到现代聊天记录文件可能包含 base64 头像等大数据 */
|
||||
@@ -172,10 +173,19 @@ export class FormatSniffer {
|
||||
* 根据格式 ID 获取解析器
|
||||
*/
|
||||
getParserById(formatId: string): Parser | null {
|
||||
const module = this.formats.find((m) => m.feature.id === formatId)
|
||||
const canonicalId = normalizeParserFormatId(formatId)
|
||||
const module = this.formats.find((m) => m.feature.id === canonicalId)
|
||||
return module?.parser || null
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve canonical and released legacy format IDs to one canonical feature.
|
||||
*/
|
||||
getFeatureById(formatId: string): FormatFeature | null {
|
||||
const canonicalId = normalizeParserFormatId(formatId)
|
||||
return this.formats.find((module) => module.feature.id === canonicalId)?.feature ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有支持的格式
|
||||
*/
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
import type { ChatPlatform, ChatType, ParsedMember, ParsedMessage } from '@openchatlab/shared-types'
|
||||
import type { ParserFormatId } from './format-ids'
|
||||
|
||||
// ==================== 标准层:统一输出结构 ====================
|
||||
|
||||
@@ -83,7 +84,7 @@ export interface FormatSignatures {
|
||||
*/
|
||||
export interface FormatFeature {
|
||||
/** 唯一标识符 */
|
||||
id: string
|
||||
id: ParserFormatId
|
||||
/** 显示名称 */
|
||||
name: string
|
||||
/** 平台 */
|
||||
|
||||
@@ -211,7 +211,7 @@ function createWhatsAppFixture(): BenchmarkFixture {
|
||||
const blob = new Blob([lines.join('\n')], { type: 'text/plain' })
|
||||
return {
|
||||
label: 'WhatsApp TXT',
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
source: blobSource('WhatsApp benchmark.txt', blob),
|
||||
firstContent: 'fixture-message-0',
|
||||
lastContent: `fixture-message-${MESSAGE_COUNT - 1}`,
|
||||
@@ -233,7 +233,7 @@ function createLineFixture(): BenchmarkFixture {
|
||||
const blob = new Blob([lines.join('\n')], { type: 'text/plain' })
|
||||
return {
|
||||
label: 'LINE TXT',
|
||||
formatId: 'line-native-txt',
|
||||
formatId: 'line-native',
|
||||
source: blobSource('[LINE] Benchmark Team.txt', blob),
|
||||
firstContent: 'fixture-message-0',
|
||||
lastContent: `fixture-message-${MESSAGE_COUNT - 1}`,
|
||||
@@ -255,7 +255,7 @@ function createQqFixture(): BenchmarkFixture {
|
||||
const blob = new Blob([lines.join('\n')], { type: 'text/plain' })
|
||||
return {
|
||||
label: 'QQ TXT',
|
||||
formatId: 'qq-native-txt',
|
||||
formatId: 'qq-native',
|
||||
source: blobSource('qq-benchmark.txt', blob),
|
||||
firstContent: 'fixture-message-0',
|
||||
lastContent: `fixture-message-${MESSAGE_COUNT - 1}`,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
PARSER_FORMAT_IDS,
|
||||
detectLineText,
|
||||
detectQqText,
|
||||
detectTelegramMultiChatJson,
|
||||
@@ -12,6 +13,7 @@ import {
|
||||
parseWhatsAppText,
|
||||
parseWeFlowJson,
|
||||
scanTelegramChatsJson,
|
||||
normalizeParserFormatId,
|
||||
type TelegramChatInfo,
|
||||
} from '@openchatlab/parser/browser'
|
||||
|
||||
@@ -28,12 +30,12 @@ import { parseWithWasm, type BrowserImportLogEvent, type BrowserWasmParserLoader
|
||||
|
||||
export type BrowserImportFormatId =
|
||||
| ChatLabBrowserFormatId
|
||||
| 'whatsapp-native-txt'
|
||||
| 'line-native-txt'
|
||||
| 'qq-native-txt'
|
||||
| 'telegram-native'
|
||||
| 'telegram-native-single'
|
||||
| 'weflow'
|
||||
| typeof PARSER_FORMAT_IDS.WHATSAPP_NATIVE
|
||||
| typeof PARSER_FORMAT_IDS.LINE_NATIVE
|
||||
| typeof PARSER_FORMAT_IDS.QQ_NATIVE
|
||||
| typeof PARSER_FORMAT_IDS.TELEGRAM_NATIVE
|
||||
| typeof PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE
|
||||
| typeof PARSER_FORMAT_IDS.WEFLOW
|
||||
|
||||
export interface ParseBrowserSourceOptions {
|
||||
formatId?: BrowserImportFormatId
|
||||
@@ -51,18 +53,35 @@ export interface BrowserImportParseResult extends Omit<BrowserChatParseResult, '
|
||||
|
||||
const HEAD_BYTES = 64 * 1024
|
||||
|
||||
const BROWSER_IMPORT_FORMAT_IDS = new Set<string>([
|
||||
PARSER_FORMAT_IDS.CHATLAB,
|
||||
PARSER_FORMAT_IDS.CHATLAB_JSONL,
|
||||
PARSER_FORMAT_IDS.WEFLOW,
|
||||
PARSER_FORMAT_IDS.WHATSAPP_NATIVE,
|
||||
PARSER_FORMAT_IDS.LINE_NATIVE,
|
||||
PARSER_FORMAT_IDS.QQ_NATIVE,
|
||||
PARSER_FORMAT_IDS.TELEGRAM_NATIVE,
|
||||
PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE,
|
||||
])
|
||||
|
||||
export function normalizeBrowserImportFormatId(formatId: string | undefined): BrowserImportFormatId | undefined {
|
||||
if (!formatId) return undefined
|
||||
const canonicalId = normalizeParserFormatId(formatId)
|
||||
return BROWSER_IMPORT_FORMAT_IDS.has(canonicalId) ? (canonicalId as BrowserImportFormatId) : undefined
|
||||
}
|
||||
|
||||
export async function detectBrowserImportFormat(source: BrowserParseSource): Promise<BrowserImportFormatId | null> {
|
||||
if (source.name.toLowerCase().endsWith('.txt')) {
|
||||
const head = await source.slice(0, HEAD_BYTES).text()
|
||||
if (detectWhatsAppText(head, source.name)) return 'whatsapp-native-txt'
|
||||
if (detectQqText(head, source.name)) return 'qq-native-txt'
|
||||
return detectLineText(head, source.name) ? 'line-native-txt' : null
|
||||
if (detectWhatsAppText(head, source.name)) return PARSER_FORMAT_IDS.WHATSAPP_NATIVE
|
||||
if (detectQqText(head, source.name)) return PARSER_FORMAT_IDS.QQ_NATIVE
|
||||
return detectLineText(head, source.name) ? PARSER_FORMAT_IDS.LINE_NATIVE : null
|
||||
}
|
||||
if (source.name.toLowerCase().endsWith('.json')) {
|
||||
const head = await source.slice(0, HEAD_BYTES).text()
|
||||
if (detectWeFlowJson(head, source.name)) return 'weflow'
|
||||
if (detectTelegramMultiChatJson(head, source.name)) return 'telegram-native'
|
||||
if (detectTelegramSingleJson(head, source.name)) return 'telegram-native-single'
|
||||
if (detectWeFlowJson(head, source.name)) return PARSER_FORMAT_IDS.WEFLOW
|
||||
if (detectTelegramMultiChatJson(head, source.name)) return PARSER_FORMAT_IDS.TELEGRAM_NATIVE
|
||||
if (detectTelegramSingleJson(head, source.name)) return PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE
|
||||
}
|
||||
return detectChatLabFormat(source)
|
||||
}
|
||||
@@ -72,7 +91,7 @@ export async function scanBrowserMultiChatSource(
|
||||
options: Pick<ParseBrowserSourceOptions, 'checkCancelled' | 'yieldEvery'> = {}
|
||||
): Promise<TelegramChatInfo[]> {
|
||||
const formatId = await detectBrowserImportFormat(source)
|
||||
if (formatId !== 'telegram-native') {
|
||||
if (formatId !== PARSER_FORMAT_IDS.TELEGRAM_NATIVE) {
|
||||
throw new WebRuntimeError('NOT_MULTI_CHAT_FORMAT', 'The selected file is not a supported multi-chat export')
|
||||
}
|
||||
options.checkCancelled?.()
|
||||
@@ -85,7 +104,9 @@ export async function parseBrowserImportSource(
|
||||
source: BrowserParseSource,
|
||||
options: ParseBrowserSourceOptions = {}
|
||||
): Promise<BrowserImportParseResult> {
|
||||
const formatId = options.formatId ?? (await detectBrowserImportFormat(source))
|
||||
const formatId = options.formatId
|
||||
? normalizeBrowserImportFormatId(options.formatId)
|
||||
: await detectBrowserImportFormat(source)
|
||||
if (!formatId) {
|
||||
throw new WebRuntimeError(
|
||||
'UNSUPPORTED_IMPORT_FORMAT',
|
||||
@@ -93,7 +114,7 @@ export async function parseBrowserImportSource(
|
||||
)
|
||||
}
|
||||
|
||||
if (formatId === 'chatlab' || formatId === 'weflow') {
|
||||
if (formatId === PARSER_FORMAT_IDS.CHATLAB || formatId === PARSER_FORMAT_IDS.WEFLOW) {
|
||||
const wasmResult = await parseWithWasm(source, formatId, {
|
||||
checkCancelled: options.checkCancelled,
|
||||
onProgress: options.onProgress,
|
||||
@@ -103,7 +124,7 @@ export async function parseBrowserImportSource(
|
||||
if (wasmResult) return wasmResult
|
||||
}
|
||||
|
||||
if (formatId === 'whatsapp-native-txt') {
|
||||
if (formatId === PARSER_FORMAT_IDS.WHATSAPP_NATIVE) {
|
||||
options.checkCancelled?.()
|
||||
const content = await source.text()
|
||||
options.checkCancelled?.()
|
||||
@@ -120,7 +141,7 @@ export async function parseBrowserImportSource(
|
||||
}
|
||||
}
|
||||
|
||||
if (formatId === 'weflow') {
|
||||
if (formatId === PARSER_FORMAT_IDS.WEFLOW) {
|
||||
options.checkCancelled?.()
|
||||
const content = await source.text()
|
||||
options.checkCancelled?.()
|
||||
@@ -137,7 +158,7 @@ export async function parseBrowserImportSource(
|
||||
}
|
||||
}
|
||||
|
||||
if (formatId === 'line-native-txt') {
|
||||
if (formatId === PARSER_FORMAT_IDS.LINE_NATIVE) {
|
||||
options.checkCancelled?.()
|
||||
const content = await source.text()
|
||||
options.checkCancelled?.()
|
||||
@@ -154,7 +175,7 @@ export async function parseBrowserImportSource(
|
||||
}
|
||||
}
|
||||
|
||||
if (formatId === 'qq-native-txt') {
|
||||
if (formatId === PARSER_FORMAT_IDS.QQ_NATIVE) {
|
||||
options.checkCancelled?.()
|
||||
const content = await source.text()
|
||||
options.checkCancelled?.()
|
||||
@@ -171,7 +192,7 @@ export async function parseBrowserImportSource(
|
||||
}
|
||||
}
|
||||
|
||||
if (formatId === 'telegram-native-single') {
|
||||
if (formatId === PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE) {
|
||||
options.checkCancelled?.()
|
||||
const content = await source.text()
|
||||
options.checkCancelled?.()
|
||||
@@ -188,7 +209,7 @@ export async function parseBrowserImportSource(
|
||||
}
|
||||
}
|
||||
|
||||
if (formatId === 'telegram-native') {
|
||||
if (formatId === PARSER_FORMAT_IDS.TELEGRAM_NATIVE) {
|
||||
if (options.chatIndex === undefined) {
|
||||
throw new WebRuntimeError('MULTI_CHAT_SELECTION_REQUIRED', 'A Telegram chat index is required for import')
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
|
||||
assert.deepEqual(result, {
|
||||
sessionId: 'whatsapp-system-participant',
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
messageCount: 1,
|
||||
memberCount: 1,
|
||||
skippedCount: 0,
|
||||
@@ -351,11 +351,11 @@ describe('BrowserSessionRuntime', () => {
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
assert.equal(await runtime.detectFormat(fixture), 'whatsapp-native-txt')
|
||||
assert.equal(await runtime.detectFormat(fixture), 'whatsapp-native')
|
||||
assert.deepEqual(
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'whatsapp-native-txt'),
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'whatsapp-native'),
|
||||
{
|
||||
id: 'whatsapp-native-txt',
|
||||
id: 'whatsapp-native',
|
||||
name: 'WhatsApp TXT',
|
||||
platform: 'whatsapp',
|
||||
extensions: ['.txt'],
|
||||
@@ -366,7 +366,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
|
||||
assert.deepEqual(result, {
|
||||
sessionId: 'whatsapp-session',
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
messageCount: 3,
|
||||
memberCount: 3,
|
||||
skippedCount: 0,
|
||||
@@ -384,7 +384,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
groupAvatar: null,
|
||||
ownerId: null,
|
||||
lastMessageTs: Math.floor(new Date('2024-01-02T03:06:00').getTime() / 1000),
|
||||
formatId: 'whatsapp-native-txt',
|
||||
formatId: 'whatsapp-native',
|
||||
},
|
||||
])
|
||||
const sessionDb = database.getDatabase(sessionDatabaseFilename('whatsapp-session'))
|
||||
@@ -514,11 +514,11 @@ describe('BrowserSessionRuntime', () => {
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
assert.equal(await runtime.detectFormat(fixture), 'line-native-txt')
|
||||
assert.equal(await runtime.detectFormat(fixture), 'line-native')
|
||||
assert.deepEqual(
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'line-native-txt'),
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'line-native'),
|
||||
{
|
||||
id: 'line-native-txt',
|
||||
id: 'line-native',
|
||||
name: 'LINE TXT',
|
||||
platform: 'line',
|
||||
extensions: ['.txt'],
|
||||
@@ -529,7 +529,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
|
||||
assert.deepEqual(result, {
|
||||
sessionId: 'line-session',
|
||||
formatId: 'line-native-txt',
|
||||
formatId: 'line-native',
|
||||
messageCount: 3,
|
||||
memberCount: 3,
|
||||
skippedCount: 0,
|
||||
@@ -541,7 +541,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
type: session.type,
|
||||
formatId: session.formatId,
|
||||
})),
|
||||
[{ name: 'Project Team', platform: 'line', type: 'group', formatId: 'line-native-txt' }]
|
||||
[{ name: 'Project Team', platform: 'line', type: 'group', formatId: 'line-native' }]
|
||||
)
|
||||
const sessionDb = database.getDatabase(sessionDatabaseFilename('line-session'))
|
||||
assert.deepEqual(sessionDb?.prepare('SELECT sender_account_name, type, content FROM message ORDER BY id').all(), [
|
||||
@@ -572,11 +572,11 @@ describe('BrowserSessionRuntime', () => {
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
assert.equal(await runtime.detectFormat(fixture), 'qq-native-txt')
|
||||
assert.equal(await runtime.detectFormat(fixture), 'qq-native')
|
||||
assert.deepEqual(
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'qq-native-txt'),
|
||||
runtime.getSupportedFormats().find((format) => format.id === 'qq-native'),
|
||||
{
|
||||
id: 'qq-native-txt',
|
||||
id: 'qq-native',
|
||||
name: 'QQ TXT',
|
||||
platform: 'qq',
|
||||
extensions: ['.txt'],
|
||||
@@ -587,7 +587,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
|
||||
assert.deepEqual(result, {
|
||||
sessionId: 'qq-session',
|
||||
formatId: 'qq-native-txt',
|
||||
formatId: 'qq-native',
|
||||
messageCount: 2,
|
||||
memberCount: 2,
|
||||
skippedCount: 0,
|
||||
@@ -599,7 +599,7 @@ describe('BrowserSessionRuntime', () => {
|
||||
type: session.type,
|
||||
formatId: session.formatId,
|
||||
})),
|
||||
[{ name: 'Project Team', platform: 'qq', type: 'group', formatId: 'qq-native-txt' }]
|
||||
[{ name: 'Project Team', platform: 'qq', type: 'group', formatId: 'qq-native' }]
|
||||
)
|
||||
const sessionDb = database.getDatabase(sessionDatabaseFilename('qq-session'))
|
||||
assert.deepEqual(sessionDb?.prepare('SELECT sender_account_name, type, content FROM message ORDER BY id').all(), [
|
||||
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
type WordFrequencyResult,
|
||||
type YearlyActivity,
|
||||
} from '@openchatlab/core'
|
||||
import { PARSER_FORMAT_IDS } from '@openchatlab/parser/browser'
|
||||
import { WebRuntimeError } from '../runtime-error'
|
||||
import type { WorkspaceDatabasePort, WorkspaceDatabaseStage } from '../storage/workspace-database'
|
||||
import {
|
||||
@@ -120,18 +121,18 @@ const SUPPORTED_FORMATS: BrowserImportFormatInfo[] = [
|
||||
{ id: 'chatlab', name: 'ChatLab JSON', platform: 'unknown', extensions: ['.json'] },
|
||||
{ id: 'chatlab-jsonl', name: 'ChatLab JSONL', platform: 'unknown', extensions: ['.jsonl'] },
|
||||
{ id: 'weflow', name: 'WeFlow JSON', platform: 'weixin', extensions: ['.json'] },
|
||||
{ id: 'whatsapp-native-txt', name: 'WhatsApp TXT', platform: 'whatsapp', extensions: ['.txt'] },
|
||||
{ id: 'line-native-txt', name: 'LINE TXT', platform: 'line', extensions: ['.txt'] },
|
||||
{ id: 'qq-native-txt', name: 'QQ TXT', platform: 'qq', extensions: ['.txt'] },
|
||||
{ id: PARSER_FORMAT_IDS.WHATSAPP_NATIVE, name: 'WhatsApp TXT', platform: 'whatsapp', extensions: ['.txt'] },
|
||||
{ id: PARSER_FORMAT_IDS.LINE_NATIVE, name: 'LINE TXT', platform: 'line', extensions: ['.txt'] },
|
||||
{ id: PARSER_FORMAT_IDS.QQ_NATIVE, name: 'QQ TXT', platform: 'qq', extensions: ['.txt'] },
|
||||
{
|
||||
id: 'telegram-native',
|
||||
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE,
|
||||
name: 'Telegram JSON',
|
||||
platform: 'telegram',
|
||||
extensions: ['.json'],
|
||||
multiChat: true,
|
||||
},
|
||||
{
|
||||
id: 'telegram-native-single',
|
||||
id: PARSER_FORMAT_IDS.TELEGRAM_NATIVE_SINGLE,
|
||||
name: 'Telegram JSON',
|
||||
platform: 'telegram',
|
||||
extensions: ['.json'],
|
||||
|
||||
@@ -14,7 +14,11 @@ export type {
|
||||
WebRuntimeTaskResult,
|
||||
WebRuntimeTaskType,
|
||||
} from './rpc/protocol'
|
||||
export type { BrowserImportFormatId, BrowserParseSource } from './import/browser-parser'
|
||||
export {
|
||||
normalizeBrowserImportFormatId,
|
||||
type BrowserImportFormatId,
|
||||
type BrowserParseSource,
|
||||
} from './import/browser-parser'
|
||||
export type { BrowserSessionCatalogItem } from './import/session-catalog'
|
||||
export type {
|
||||
BrowserImportFormatInfo,
|
||||
|
||||
@@ -295,7 +295,7 @@ describe('BrowserImportAdapter', () => {
|
||||
assert.deepEqual(deletedSessionIds, ['session-1'])
|
||||
})
|
||||
|
||||
it('forwards browser-safe format identifiers to the worker runtime', async () => {
|
||||
it('normalizes released browser format aliases before forwarding them to the worker runtime', async () => {
|
||||
const requestedFormats: string[] = []
|
||||
const rpc = {
|
||||
async request<T extends WebRuntimeTaskType>(
|
||||
@@ -347,9 +347,9 @@ describe('BrowserImportAdapter', () => {
|
||||
assert.equal(telegram.success, true)
|
||||
assert.deepEqual(requestedFormats, [
|
||||
'weflow',
|
||||
'whatsapp-native-txt',
|
||||
'line-native-txt',
|
||||
'qq-native-txt',
|
||||
'whatsapp-native',
|
||||
'line-native',
|
||||
'qq-native',
|
||||
'telegram-native-single',
|
||||
])
|
||||
})
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import type { BrowserImportFormatId, BrowserParseSource, RpcProgressPayload } from '@openchatlab/web-runtime'
|
||||
import {
|
||||
normalizeBrowserImportFormatId,
|
||||
type BrowserImportFormatId,
|
||||
type BrowserParseSource,
|
||||
type RpcProgressPayload,
|
||||
} from '@openchatlab/web-runtime'
|
||||
import { rebaseChatLabDemoDocuments } from '@openchatlab/parser/browser'
|
||||
import { reportRuntimeLog } from '@/services/log-report'
|
||||
import type { BrowserRuntimeRpcPort } from '../browser-runtime/types'
|
||||
@@ -38,7 +43,7 @@ export class BrowserImportAdapter implements ImportAdapter {
|
||||
if (typeof file === 'string') return { success: false, error: 'File path import is not available in Web WASM' }
|
||||
if (this.activeImport) return { success: false, error: 'Another Web WASM import is already running' }
|
||||
|
||||
const formatId = normalizeFormatId(options?.formatId)
|
||||
const formatId = normalizeBrowserImportFormatId(options?.formatId)
|
||||
if (options?.formatId && !formatId) {
|
||||
return { success: false, error: `Unsupported Web WASM import format: ${options.formatId}` }
|
||||
}
|
||||
@@ -281,19 +286,6 @@ function mapImportProgress(progress: RpcProgressPayload): ImportProgress {
|
||||
return mapped
|
||||
}
|
||||
|
||||
function normalizeFormatId(formatId: string | undefined): BrowserImportFormatId | undefined {
|
||||
return formatId === 'chatlab' ||
|
||||
formatId === 'chatlab-jsonl' ||
|
||||
formatId === 'weflow' ||
|
||||
formatId === 'whatsapp-native-txt' ||
|
||||
formatId === 'line-native-txt' ||
|
||||
formatId === 'qq-native-txt' ||
|
||||
formatId === 'telegram-native' ||
|
||||
formatId === 'telegram-native-single'
|
||||
? formatId
|
||||
: undefined
|
||||
}
|
||||
|
||||
function unsupported<T>(capability: string): Promise<T> {
|
||||
return Promise.reject(new Error(`${capability} is not available in Web WASM`))
|
||||
}
|
||||
@@ -134,7 +134,7 @@ describe('archive import source adapters', () => {
|
||||
success: true,
|
||||
source: {
|
||||
sourceId: 'source-1',
|
||||
formatId: 'google-chat-takeout',
|
||||
formatId: 'google-chat-native',
|
||||
platform: 'google-chat',
|
||||
chats: [],
|
||||
expiresAt: 123,
|
||||
@@ -184,7 +184,7 @@ describe('archive import source adapters', () => {
|
||||
success: true,
|
||||
source: {
|
||||
sourceId: 'source-2',
|
||||
formatId: 'google-chat-takeout',
|
||||
formatId: 'google-chat-native',
|
||||
platform: 'google-chat',
|
||||
chats: [],
|
||||
expiresAt: 456,
|
||||
|
||||
Reference in new issue
Block a user