diff --git a/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts b/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts new file mode 100644 index 00000000..a2a23ac0 --- /dev/null +++ b/packages/parser/src/formats/shuakami-qq-exporter-chunked.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, it } from 'node:test' +import { ChatType, MessageType } from '@openchatlab/shared-types' + +import { detectFormat, findEntryFileInDirectory, parseFileSync } from '../index' + +describe('shuakami-qq-exporter chunked parser', () => { + it('detects and parses manifests that store the JSONL format in chunked', async () => { + const root = mkdtempSync(join(tmpdir(), 'chatlab-qce-chunked-')) + const chunksDir = join(root, 'chunks') + const manifestPath = join(root, 'manifest.json') + + try { + mkdirSync(chunksDir) + writeFileSync( + manifestPath, + JSON.stringify({ + metadata: { name: 'QQChatExporter', version: '0.1.0' }, + chatInfo: { name: 'Test Group', type: 'group', selfUin: '100' }, + statistics: { totalMessages: 2 }, + chunked: { + format: 'jsonl', + chunksDir: 'chunks', + chunkFileExt: '.jsonl', + maxMessagesPerChunk: 50000, + maxBytesPerChunk: 52428800, + chunks: [{ relativePath: 'chunks/c000001.jsonl', count: 2 }], + }, + }) + ) + writeFileSync( + join(chunksDir, 'c000001.jsonl'), + [ + { + id: 'message-1', + timestamp: 1752192000000, + sender: { uid: 'u_100', uin: '100', name: 'Alice', nickname: 'Alice' }, + type: 'text', + content: { text: 'hello' }, + }, + { + id: 'message-2', + timestamp: 1752192060000, + sender: { uid: 'u_200', uin: '200', name: 'Bob', nickname: 'Bob' }, + type: 'text', + content: { text: 'recalled' }, + recalled: true, + }, + ] + .map((message) => JSON.stringify(message)) + .join('\n') + ) + + assert.equal(detectFormat(manifestPath)?.id, 'shuakami-qq-exporter-chunked') + assert.equal(findEntryFileInDirectory(root), manifestPath) + + const result = await parseFileSync(manifestPath) + assert.equal(result.meta.type, ChatType.GROUP) + assert.equal(result.messages.length, 2) + assert.equal(result.messages[1].type, MessageType.RECALL) + assert.equal(result.messages[1].content, '[已撤回] recalled') + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts b/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts index 0c2e4c43..4bc198d0 100644 --- a/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts +++ b/packages/parser/src/formats/shuakami-qq-exporter-chunked.ts @@ -48,8 +48,8 @@ export const feature: FormatFeature = { priority: 5, // 比单文件版本优先级更高 extensions: ['.json'], signatures: { - // 使用 head 签名 "format": "chunked-jsonl" 来区分。 - head: [/"format"\s*:\s*"chunked-jsonl"/], + // V5 使用 metadata.format,0.1.x 将格式标识移至 chunked.format。 + head: [/"format"\s*:\s*"chunked-jsonl"/, /"chunked"\s*:\s*\{[^{}]*"format"\s*:\s*"jsonl"/], requiredFields: ['metadata', 'chatInfo'], }, } @@ -74,9 +74,9 @@ interface Manifest { metadata: { name?: string copyright?: string - exportTime: string + exportTime?: string version: string - format: string + format?: string } chatInfo: { name: string @@ -339,9 +339,10 @@ async function* parseChunkedJsonl(options: ParseOptions): AsyncGenerator