diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index bd14fc1d..c4d7c740 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -16,6 +16,7 @@ import { initRuntime, resolveNativeBinding } from './runtime' import { registerQueryCommands } from './query/register' import { registerManifestCommand } from './query/manifest' import { registerImportCommand } from './import/command' +import { registerValidateCommand } from './validate/command' const program = new Command() @@ -51,6 +52,7 @@ registerQueryCommands(program) registerManifestCommand(program, getVersion()) registerImportCommand(program) +registerValidateCommand(program) program .command('formats') diff --git a/apps/cli/src/validate/command.ts b/apps/cli/src/validate/command.ts new file mode 100644 index 00000000..d74808e5 --- /dev/null +++ b/apps/cli/src/validate/command.ts @@ -0,0 +1,127 @@ +import fs from 'node:fs' +import type { Command } from 'commander' +import { validateChatLabFile, type ChatLabValidationReport } from '@openchatlab/parser' +import { appLogger } from '@openchatlab/node-runtime' + +interface ValidateCommandOptions { + json?: boolean +} + +interface ValidateCommandEnvelope { + ok: boolean + command: 'validate' + data?: ChatLabValidationReport + error?: { + code: string + message: string + hint?: string + } + meta: { + apiVersion: 1 + } +} + +function buildEnvelope(report: ChatLabValidationReport): ValidateCommandEnvelope { + return report.valid + ? { ok: true, command: 'validate', data: report, meta: { apiVersion: 1 } } + : { + ok: false, + command: 'validate', + data: report, + error: { + code: 'INVALID_CHATLAB_FORMAT', + message: `Validation found ${report.errorCount} error(s).`, + hint: 'Fix every error, rerun validation, then preview the import with "chatlab import --dry-run --json".', + }, + meta: { apiVersion: 1 }, + } +} + +function buildFileError(file: string): ValidateCommandEnvelope { + return { + ok: false, + command: 'validate', + error: { + code: 'FILE_NOT_FOUND', + message: `File not found: ${file}`, + hint: 'Pass an absolute path or a path relative to the current working directory.', + }, + meta: { apiVersion: 1 }, + } +} + +function printHumanReport(report: ChatLabValidationReport): void { + const status = report.valid ? 'Validation passed.' : 'Validation failed.' + console.log(status) + console.log(` Format: ${report.format}`) + if (report.version) console.log(` Version: ${report.version}`) + console.log(` Members: ${report.stats.memberCount}`) + console.log(` Messages: ${report.stats.messageCount}`) + console.log(` Errors: ${report.errorCount}`) + console.log(` Warnings: ${report.warningCount}`) + + for (const issue of report.issues) { + const location = [issue.line ? `line ${issue.line}` : '', issue.path ?? ''].filter(Boolean).join(', ') + console.log( + ` [${issue.severity.toUpperCase()}] ${issue.code}${location ? ` (${location})` : ''}: ${issue.message}` + ) + } + if (report.truncatedIssueCount > 0) { + console.log(` ... ${report.truncatedIssueCount} more issue(s) not shown`) + } +} + +export function registerValidateCommand(program: Command): void { + program + .command('validate ') + .description('Strictly validate a ChatLab JSON or JSONL file without importing it') + .option('--json', 'Output one machine-readable JSON envelope') + .action(async (file: string, options: ValidateCommandOptions) => { + if (!fs.existsSync(file)) { + const envelope = buildFileError(file) + if (options.json) process.stdout.write(`${JSON.stringify(envelope)}\n`) + else { + console.error(envelope.error?.message) + if (envelope.error?.hint) console.error(` Hint: ${envelope.error.hint}`) + } + process.exitCode = 3 + return + } + + try { + appLogger.info('validate', 'ChatLab format validation started', { + extension: file.toLowerCase().endsWith('.jsonl') + ? '.jsonl' + : file.toLowerCase().endsWith('.json') + ? '.json' + : 'other', + fileSize: fs.statSync(file).size, + }) + const report = await validateChatLabFile(file) + const envelope = buildEnvelope(report) + if (options.json) process.stdout.write(`${JSON.stringify(envelope)}\n`) + else printHumanReport(report) + + appLogger.info('validate', 'ChatLab format validation completed', { + valid: report.valid, + errors: report.errorCount, + warnings: report.warningCount, + members: report.stats.memberCount, + messages: report.stats.messageCount, + }) + if (!report.valid) process.exitCode = 1 + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const envelope: ValidateCommandEnvelope = { + ok: false, + command: 'validate', + error: { code: 'VALIDATION_FAILED', message }, + meta: { apiVersion: 1 }, + } + if (options.json) process.stdout.write(`${JSON.stringify(envelope)}\n`) + else console.error(`Validation failed: ${message}`) + appLogger.error('validate', 'ChatLab format validation failed', error) + process.exitCode = 1 + } + }) +} diff --git a/packages/parser/src/chatlab-validator.test.ts b/packages/parser/src/chatlab-validator.test.ts new file mode 100644 index 00000000..2b21dfbf --- /dev/null +++ b/packages/parser/src/chatlab-validator.test.ts @@ -0,0 +1,242 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { CHATLAB_FORMAT_VERSION } from '@openchatlab/shared-types' +import { validateChatLabFile } from './chatlab-validator' + +function makeTempDir(): string { + const baseDir = process.env.CHATLAB_TEST_TMPDIR ?? os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-validator-')) +} + +function writeJsonl(filePath: string, rows: unknown[]): void { + fs.writeFileSync(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8') +} + +test('validates a complete ChatLab JSONL file without exposing message content', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'converted.jsonl') + fs.writeFileSync( + filePath, + [ + '# generated locally', + '', + JSON.stringify({ + _type: 'header', + chatlab: { version: CHATLAB_FORMAT_VERSION, exportedAt: 1_711_468_800 }, + meta: { name: 'Example', platform: 'custom', type: 'group', ownerId: 'alice' }, + }), + JSON.stringify({ _type: 'member', platformId: 'alice', accountName: 'Alice' }), + JSON.stringify({ _type: 'member', platformId: 'bob', accountName: 'Bob' }), + JSON.stringify({ + _type: 'message', + platformMessageId: 'message-1', + sender: 'alice', + accountName: 'Alice', + timestamp: 1_711_468_800, + type: 0, + content: 'private sample text', + }), + JSON.stringify({ + _type: 'message', + platformMessageId: 'message-2', + replyToMessageId: 'message-1', + sender: 'bob', + accountName: 'Bob', + timestamp: 1_711_468_810, + type: 25, + content: 'reply', + }), + ].join('\n'), + 'utf8' + ) + + try { + const report = await validateChatLabFile(filePath) + assert.equal(report.valid, true) + assert.equal(report.errorCount, 0) + assert.equal(report.warningCount, 0) + assert.deepEqual(report.stats, { + headerCount: 1, + memberCount: 2, + messageCount: 2, + uniqueSenderCount: 2, + commentLineCount: 1, + blankLineCount: 1, + }) + assert.equal(JSON.stringify(report).includes('private sample text'), false) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('accepts the initial ChatLab JSON format version', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'legacy.json') + fs.writeFileSync( + filePath, + JSON.stringify({ + chatlab: { version: '0.0.1', exportedAt: 1_711_468_800 }, + meta: { name: 'Legacy chat', platform: 'custom', type: 'private' }, + members: [{ platformId: 'alice', accountName: 'Alice' }], + messages: [ + { + sender: 'alice', + accountName: 'Alice', + timestamp: 1_711_468_800, + type: 0, + content: 'Hello', + }, + ], + }), + 'utf8' + ) + + try { + const report = await validateChatLabFile(filePath) + assert.equal(report.valid, true) + assert.equal(report.version, '0.0.1') + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('reports malformed and lossy JSONL output as invalid', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'broken.jsonl') + fs.writeFileSync( + filePath, + [ + JSON.stringify({ + _type: 'header', + chatlab: { version: '9.9.9', exportedAt: 1_711_468_800_000 }, + meta: { name: 'Broken', platform: 'custom', type: 'room', ownerId: 'missing-owner' }, + }), + JSON.stringify({ _type: 'member', platformId: 'alice', accountName: 'Alice' }), + '{bad json', + JSON.stringify({ + _type: 'message', + platformMessageId: 'same-id', + sender: 'missing-member', + accountName: 'Unknown', + timestamp: 1_711_468_800_000, + type: 6, + content: 'text', + }), + JSON.stringify({ + _type: 'message', + platformMessageId: 'same-id', + replyToMessageId: 'not-found', + sender: 'alice', + accountName: 'Alice', + timestamp: 1_711_468_700, + type: 0, + content: 'text', + }), + JSON.stringify({ _type: 'member', platformId: 'late', accountName: 'Late member' }), + ].join('\n'), + 'utf8' + ) + + try { + const report = await validateChatLabFile(filePath) + const codes = new Set(report.issues.map((issue) => issue.code)) + assert.equal(report.valid, false) + assert.ok(report.errorCount >= 7) + assert.ok(codes.has('UNSUPPORTED_VERSION')) + assert.ok(codes.has('INVALID_UNIX_SECONDS')) + assert.ok(codes.has('INVALID_CHAT_TYPE')) + assert.ok(codes.has('INVALID_JSONL_LINE')) + assert.ok(codes.has('INVALID_MESSAGE_TYPE')) + assert.ok(codes.has('DUPLICATE_MESSAGE_ID')) + assert.ok(codes.has('UNKNOWN_MESSAGE_SENDER')) + assert.ok(codes.has('UNKNOWN_OWNER')) + assert.ok(codes.has('MEMBER_AFTER_MESSAGE')) + assert.ok(codes.has('UNKNOWN_REPLY_TARGET')) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('validates ChatLab JSON members and messages', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'converted.json') + fs.writeFileSync( + filePath, + JSON.stringify({ + chatlab: { version: CHATLAB_FORMAT_VERSION, exportedAt: 1_711_468_800 }, + meta: { name: 'Direct chat', platform: 'custom', type: 'private' }, + members: [ + { platformId: 'me', accountName: 'Me' }, + { platformId: 'friend', accountName: 'Friend', aliases: ['F'] }, + ], + messages: [ + { + sender: 'me', + accountName: 'Me', + timestamp: 1_711_468_800, + type: 0, + content: 'Hello', + }, + ], + }), + 'utf8' + ) + + try { + const report = await validateChatLabFile(filePath) + assert.equal(report.valid, true) + assert.equal(report.format, 'json') + assert.equal(report.stats.memberCount, 2) + assert.equal(report.stats.messageCount, 1) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('requires explicit members in ChatLab JSON', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'missing-members.json') + fs.writeFileSync( + filePath, + JSON.stringify({ + chatlab: { version: CHATLAB_FORMAT_VERSION, exportedAt: 1_711_468_800 }, + meta: { name: 'Direct chat', platform: 'custom', type: 'private' }, + messages: [ + { + sender: 'me', + accountName: 'Me', + timestamp: 1_711_468_800, + type: 0, + content: 'Hello', + }, + ], + }), + 'utf8' + ) + + try { + const report = await validateChatLabFile(filePath) + assert.equal(report.valid, false) + assert.ok(report.issues.some((issue) => issue.code === 'MISSING_MEMBERS')) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('rejects unsupported output extensions', async () => { + const root = makeTempDir() + const filePath = path.join(root, 'converted.txt') + writeJsonl(filePath, []) + + try { + const report = await validateChatLabFile(filePath) + assert.equal(report.valid, false) + assert.equal(report.errorCount, 1) + assert.ok(report.issues.some((issue) => issue.code === 'UNSUPPORTED_EXTENSION')) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/packages/parser/src/chatlab-validator.ts b/packages/parser/src/chatlab-validator.ts new file mode 100644 index 00000000..d7227b3c --- /dev/null +++ b/packages/parser/src/chatlab-validator.ts @@ -0,0 +1,523 @@ +import { createReadStream } from 'node:fs' +import { readFile, stat } from 'node:fs/promises' +import { extname } from 'node:path' +import { createInterface } from 'node:readline' +import { + CHATLAB_SUPPORTED_FORMAT_VERSIONS, + MessageType, + isSupportedChatLabFormatVersion, +} from '@openchatlab/shared-types' + +const MAX_REPORTED_ISSUES = 100 +const MAX_UNIX_SECONDS = 100_000_000_000 +const KNOWN_MESSAGE_TYPES = new Set( + Object.values(MessageType).filter((value): value is number => typeof value === 'number') +) + +export type ChatLabValidationFormat = 'json' | 'jsonl' | 'unknown' +export type ChatLabValidationSeverity = 'error' | 'warning' + +export interface ChatLabValidationIssue { + severity: ChatLabValidationSeverity + code: string + message: string + path?: string + line?: number +} + +export interface ChatLabValidationStats { + headerCount: number + memberCount: number + messageCount: number + uniqueSenderCount: number + commentLineCount: number + blankLineCount: number +} + +export interface ChatLabValidationReport { + valid: boolean + format: ChatLabValidationFormat + version?: string + fileSize: number + errorCount: number + warningCount: number + issues: ChatLabValidationIssue[] + truncatedIssueCount: number + stats: ChatLabValidationStats +} + +interface ValidationState { + format: ChatLabValidationFormat + fileSize: number + version?: string + errorCount: number + warningCount: number + issues: ChatLabValidationIssue[] + stats: ChatLabValidationStats + memberIds: Set + ownerIds: Set + senderIds: Set + messageIds: Set + replyTargetIds: Set + lastTimestamp?: number +} + +interface IssueLocation { + path?: string + line?: number +} + +function createState(format: ChatLabValidationFormat, fileSize: number): ValidationState { + return { + format, + fileSize, + errorCount: 0, + warningCount: 0, + issues: [], + stats: { + headerCount: 0, + memberCount: 0, + messageCount: 0, + uniqueSenderCount: 0, + commentLineCount: 0, + blankLineCount: 0, + }, + memberIds: new Set(), + ownerIds: new Set(), + senderIds: new Set(), + messageIds: new Set(), + replyTargetIds: new Set(), + } +} + +function addIssue( + state: ValidationState, + severity: ChatLabValidationSeverity, + code: string, + message: string, + location: IssueLocation = {} +): void { + if (severity === 'error') state.errorCount++ + else state.warningCount++ + + if (state.issues.length < MAX_REPORTED_ISSUES) { + state.issues.push({ severity, code, message, ...location }) + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function hasOwn(value: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(value, key) +} + +function childLocation(location: IssueLocation, path: string): IssueLocation { + return { ...location, path } +} + +function validateRequiredString( + state: ValidationState, + value: unknown, + path: string, + location: IssueLocation +): value is string { + if (typeof value === 'string' && value.trim().length > 0) return true + addIssue(state, 'error', 'INVALID_REQUIRED_STRING', 'Expected a non-empty string.', childLocation(location, path)) + return false +} + +function validateOptionalString( + state: ValidationState, + object: Record, + key: string, + path: string, + location: IssueLocation +): void { + if (!hasOwn(object, key)) return + if (typeof object[key] !== 'string') { + addIssue(state, 'error', 'INVALID_OPTIONAL_STRING', 'Expected a string.', childLocation(location, path)) + } +} + +function validateUnixSeconds( + state: ValidationState, + value: unknown, + path: string, + location: IssueLocation +): value is number { + if (typeof value === 'number' && Number.isInteger(value) && value >= 0 && value < MAX_UNIX_SECONDS) { + return true + } + addIssue( + state, + 'error', + 'INVALID_UNIX_SECONDS', + 'Expected a non-negative integer Unix timestamp in seconds, not milliseconds.', + childLocation(location, path) + ) + return false +} + +function validateHeader( + state: ValidationState, + chatlab: unknown, + meta: unknown, + location: IssueLocation, + basePath = '' +): void { + state.stats.headerCount++ + + if (!isRecord(chatlab)) { + addIssue(state, 'error', 'INVALID_CHATLAB_HEADER', 'Expected a chatlab object.', { + ...location, + path: `${basePath}chatlab`, + }) + } else { + const versionPath = `${basePath}chatlab.version` + if (typeof chatlab.version !== 'string') { + addIssue(state, 'error', 'INVALID_VERSION', 'Expected a string format version.', { + ...location, + path: versionPath, + }) + } else { + state.version = chatlab.version + if (!isSupportedChatLabFormatVersion(chatlab.version)) { + addIssue( + state, + 'error', + 'UNSUPPORTED_VERSION', + `Supported ChatLab format versions: ${CHATLAB_SUPPORTED_FORMAT_VERSIONS.join(', ')}.`, + { + ...location, + path: versionPath, + } + ) + } + } + validateUnixSeconds(state, chatlab.exportedAt, `${basePath}chatlab.exportedAt`, location) + validateOptionalString(state, chatlab, 'generator', `${basePath}chatlab.generator`, location) + validateOptionalString(state, chatlab, 'description', `${basePath}chatlab.description`, location) + } + + if (!isRecord(meta)) { + addIssue(state, 'error', 'INVALID_META', 'Expected a meta object.', { + ...location, + path: `${basePath}meta`, + }) + return + } + + validateRequiredString(state, meta.name, `${basePath}meta.name`, location) + validateRequiredString(state, meta.platform, `${basePath}meta.platform`, location) + if (meta.type !== 'group' && meta.type !== 'private') { + addIssue(state, 'error', 'INVALID_CHAT_TYPE', 'Expected chat type "group" or "private".', { + ...location, + path: `${basePath}meta.type`, + }) + } + validateOptionalString(state, meta, 'groupId', `${basePath}meta.groupId`, location) + validateOptionalString(state, meta, 'groupAvatar', `${basePath}meta.groupAvatar`, location) + if (hasOwn(meta, 'ownerId')) { + if (validateRequiredString(state, meta.ownerId, `${basePath}meta.ownerId`, location)) { + state.ownerIds.add(meta.ownerId) + } + } + validateOptionalString(state, meta, 'sourceSessionId', `${basePath}meta.sourceSessionId`, location) +} + +function validateMember(state: ValidationState, value: unknown, path: string, location: IssueLocation): void { + state.stats.memberCount++ + if (!isRecord(value)) { + addIssue(state, 'error', 'INVALID_MEMBER', 'Expected a member object.', childLocation(location, path)) + return + } + + if (validateRequiredString(state, value.platformId, `${path}.platformId`, location)) { + if (state.memberIds.has(value.platformId)) { + addIssue(state, 'error', 'DUPLICATE_MEMBER_ID', 'Member platformId must be unique.', { + ...location, + path: `${path}.platformId`, + }) + } else { + state.memberIds.add(value.platformId) + } + } + validateRequiredString(state, value.accountName, `${path}.accountName`, location) + validateOptionalString(state, value, 'groupNickname', `${path}.groupNickname`, location) + validateOptionalString(state, value, 'avatar', `${path}.avatar`, location) + + if (hasOwn(value, 'aliases')) { + if (!Array.isArray(value.aliases) || value.aliases.some((alias) => typeof alias !== 'string')) { + addIssue(state, 'error', 'INVALID_ALIASES', 'Expected aliases to be an array of strings.', { + ...location, + path: `${path}.aliases`, + }) + } + } + + if (hasOwn(value, 'roles')) { + if (!Array.isArray(value.roles)) { + addIssue(state, 'error', 'INVALID_ROLES', 'Expected roles to be an array.', { + ...location, + path: `${path}.roles`, + }) + } else { + value.roles.forEach((role, index) => { + const rolePath = `${path}.roles[${index}]` + if (!isRecord(role)) { + addIssue(state, 'error', 'INVALID_ROLE', 'Expected a role object.', { + ...location, + path: rolePath, + }) + return + } + validateRequiredString(state, role.id, `${rolePath}.id`, location) + validateOptionalString(state, role, 'name', `${rolePath}.name`, location) + }) + } + } +} + +function validateMessage(state: ValidationState, value: unknown, path: string, location: IssueLocation): void { + state.stats.messageCount++ + if (!isRecord(value)) { + addIssue(state, 'error', 'INVALID_MESSAGE', 'Expected a message object.', childLocation(location, path)) + return + } + + if (validateRequiredString(state, value.sender, `${path}.sender`, location)) { + state.senderIds.add(value.sender) + } + validateRequiredString(state, value.accountName, `${path}.accountName`, location) + + if (validateUnixSeconds(state, value.timestamp, `${path}.timestamp`, location)) { + if (state.lastTimestamp !== undefined && value.timestamp < state.lastTimestamp) { + addIssue( + state, + 'warning', + 'OUT_OF_ORDER_TIMESTAMP', + 'Messages should be ordered by timestamp ascending.', + childLocation(location, `${path}.timestamp`) + ) + } + state.lastTimestamp = value.timestamp + } + + if (typeof value.type !== 'number' || !Number.isInteger(value.type) || !KNOWN_MESSAGE_TYPES.has(value.type)) { + addIssue(state, 'error', 'INVALID_MESSAGE_TYPE', 'Expected a supported numeric ChatLab message type.', { + ...location, + path: `${path}.type`, + }) + } + + if (!hasOwn(value, 'content') || (typeof value.content !== 'string' && value.content !== null)) { + addIssue(state, 'error', 'INVALID_MESSAGE_CONTENT', 'Expected content to be a string or null.', { + ...location, + path: `${path}.content`, + }) + } + validateOptionalString(state, value, 'groupNickname', `${path}.groupNickname`, location) + + if (hasOwn(value, 'platformMessageId')) { + if (validateRequiredString(state, value.platformMessageId, `${path}.platformMessageId`, location)) { + if (state.messageIds.has(value.platformMessageId)) { + addIssue(state, 'error', 'DUPLICATE_MESSAGE_ID', 'platformMessageId must be unique.', { + ...location, + path: `${path}.platformMessageId`, + }) + } else { + state.messageIds.add(value.platformMessageId) + } + } + } + + if (hasOwn(value, 'replyToMessageId')) { + if (validateRequiredString(state, value.replyToMessageId, `${path}.replyToMessageId`, location)) { + state.replyTargetIds.add(value.replyToMessageId) + } + } +} + +function finalizeState( + state: ValidationState, + requireExplicitMembers: boolean, + enforceRequiredSections = true +): ChatLabValidationReport { + if (enforceRequiredSections) { + if (state.stats.headerCount === 0) { + addIssue(state, 'error', 'MISSING_HEADER', 'A ChatLab header is required.') + } + if (state.stats.messageCount === 0) { + addIssue(state, 'error', 'MISSING_MESSAGES', 'At least one message is required.') + } + if (requireExplicitMembers && state.stats.memberCount === 0) { + addIssue(state, 'error', 'MISSING_MEMBERS', 'The JSON format requires a members array with sender identities.') + } + } + + if (state.memberIds.size > 0) { + for (const ownerId of state.ownerIds) { + if (!state.memberIds.has(ownerId)) { + addIssue(state, 'error', 'UNKNOWN_OWNER', 'meta.ownerId must reference an explicit member platformId.') + } + } + for (const senderId of state.senderIds) { + if (!state.memberIds.has(senderId)) { + addIssue( + state, + 'error', + 'UNKNOWN_MESSAGE_SENDER', + 'Every message sender must reference an explicit member platformId.' + ) + } + } + } + + for (const replyTargetId of state.replyTargetIds) { + if (!state.messageIds.has(replyTargetId)) { + addIssue( + state, + 'warning', + 'UNKNOWN_REPLY_TARGET', + 'A replyToMessageId does not match any platformMessageId in this file.' + ) + } + } + + state.stats.uniqueSenderCount = state.senderIds.size + const totalIssueCount = state.errorCount + state.warningCount + return { + valid: state.errorCount === 0, + format: state.format, + version: state.version, + fileSize: state.fileSize, + errorCount: state.errorCount, + warningCount: state.warningCount, + issues: state.issues, + truncatedIssueCount: Math.max(0, totalIssueCount - state.issues.length), + stats: state.stats, + } +} + +async function validateJson(filePath: string, fileSize: number): Promise { + const state = createState('json', fileSize) + let root: unknown + + try { + root = JSON.parse(await readFile(filePath, 'utf8')) as unknown + } catch { + addIssue(state, 'error', 'INVALID_JSON', 'The file is not valid JSON.') + return finalizeState(state, true, false) + } + + if (!isRecord(root)) { + addIssue(state, 'error', 'INVALID_ROOT', 'Expected a top-level JSON object.') + return finalizeState(state, true, false) + } + + validateHeader(state, root.chatlab, root.meta, {}) + + if (!Array.isArray(root.members)) { + addIssue(state, 'error', 'INVALID_MEMBERS', 'Expected members to be an array.', { path: 'members' }) + } else { + root.members.forEach((member, index) => validateMember(state, member, `members[${index}]`, {})) + } + + if (!Array.isArray(root.messages)) { + addIssue(state, 'error', 'INVALID_MESSAGES', 'Expected messages to be an array.', { path: 'messages' }) + } else { + root.messages.forEach((message, index) => validateMessage(state, message, `messages[${index}]`, {})) + } + + return finalizeState(state, true) +} + +async function validateJsonl(filePath: string, fileSize: number): Promise { + const state = createState('jsonl', fileSize) + const input = createReadStream(filePath, { encoding: 'utf8' }) + const lines = createInterface({ input, crlfDelay: Infinity }) + let lineNumber = 0 + let dataLineCount = 0 + let sawMessage = false + + for await (const line of lines) { + lineNumber++ + const trimmed = line.trim() + if (!trimmed) { + state.stats.blankLineCount++ + continue + } + if (trimmed.startsWith('#')) { + state.stats.commentLineCount++ + continue + } + + dataLineCount++ + let value: unknown + try { + value = JSON.parse(trimmed) as unknown + } catch { + addIssue(state, 'error', 'INVALID_JSONL_LINE', 'The line is not valid JSON.', { line: lineNumber }) + continue + } + + if (!isRecord(value)) { + addIssue(state, 'error', 'INVALID_JSONL_VALUE', 'Expected a JSON object on every data line.', { + line: lineNumber, + }) + continue + } + + if (dataLineCount === 1 && value._type !== 'header') { + addIssue(state, 'error', 'HEADER_NOT_FIRST', 'The first data line must be a header.', { + line: lineNumber, + path: '_type', + }) + } + + switch (value._type) { + case 'header': + if (state.stats.headerCount > 0) { + addIssue(state, 'error', 'DUPLICATE_HEADER', 'Only one header line is allowed.', { + line: lineNumber, + path: '_type', + }) + } + validateHeader(state, value.chatlab, value.meta, { line: lineNumber }) + break + case 'member': + if (sawMessage) { + addIssue(state, 'error', 'MEMBER_AFTER_MESSAGE', 'Member lines must appear before message lines.', { + line: lineNumber, + }) + } + validateMember(state, value, `line[${lineNumber}]`, { line: lineNumber }) + break + case 'message': + sawMessage = true + validateMessage(state, value, `line[${lineNumber}]`, { line: lineNumber }) + break + default: + addIssue(state, 'error', 'UNKNOWN_LINE_TYPE', 'Expected _type to be header, member, or message.', { + line: lineNumber, + path: '_type', + }) + } + } + + return finalizeState(state, false) +} + +export async function validateChatLabFile(filePath: string): Promise { + const fileStat = await stat(filePath) + const extension = extname(filePath).toLowerCase() + + if (extension === '.json') return validateJson(filePath, fileStat.size) + if (extension === '.jsonl') return validateJsonl(filePath, fileStat.size) + + const state = createState('unknown', fileStat.size) + addIssue(state, 'error', 'UNSUPPORTED_EXTENSION', 'Expected a .json or .jsonl file.') + return finalizeState(state, false, false) +} diff --git a/packages/parser/src/formats/chatlab-jsonl.test.ts b/packages/parser/src/formats/chatlab-jsonl.test.ts index aeb1b364..058d8eae 100644 --- a/packages/parser/src/formats/chatlab-jsonl.test.ts +++ b/packages/parser/src/formats/chatlab-jsonl.test.ts @@ -23,7 +23,9 @@ function writeChatLabJsonl(filePath: string, messageCount: number): void { _type: 'member', platformId: 'member-1', accountName: 'Alice', + aliases: ['A'], avatar: 'data:image/png;base64,AAAA', + roles: [{ id: 'owner' }], }), ] @@ -68,6 +70,8 @@ test('ChatLab JSONL emits message batches and progress while reading', async () } else if (event.type === 'members') { dataEventTypes.push(event.type) assert.equal(event.data[0]?.avatar, 'data:image/png;base64,AAAA') + assert.deepEqual(event.data[0]?.aliases, ['A']) + assert.deepEqual(event.data[0]?.roles, [{ id: 'owner' }]) } } diff --git a/packages/parser/src/formats/chatlab-jsonl.ts b/packages/parser/src/formats/chatlab-jsonl.ts index bd963590..26f212c4 100644 --- a/packages/parser/src/formats/chatlab-jsonl.ts +++ b/packages/parser/src/formats/chatlab-jsonl.ts @@ -61,6 +61,7 @@ interface JsonlMember { groupNickname?: string aliases?: string[] avatar?: string + roles?: Array<{ id: string; name?: string }> } /** Message 行结构 */ @@ -186,7 +187,9 @@ async function* parseChatLabJsonl(options: ParseOptions): AsyncGenerator