diff --git a/packages/node-runtime/package.json b/packages/node-runtime/package.json index 9cb40c5b..f5ac50bc 100644 --- a/packages/node-runtime/package.json +++ b/packages/node-runtime/package.json @@ -17,12 +17,14 @@ "onnxruntime-common": "1.24.3", "onnxruntime-node": "1.24.3", "sharp": "0.34.5", - "sqlite-vec": "^0.1.9" + "sqlite-vec": "^0.1.9", + "yauzl": "^3.4.0" }, "optionalDependencies": { "@node-rs/jieba": "^2.0.1" }, "devDependencies": { - "@types/better-sqlite3": "^7.6.13" + "@types/better-sqlite3": "^7.6.13", + "@types/yauzl": "^3.4.0" } } diff --git a/packages/parser/src/formats/google-chat-takeout.test.ts b/packages/parser/src/formats/google-chat-takeout.test.ts new file mode 100644 index 00000000..f1a7744f --- /dev/null +++ b/packages/parser/src/formats/google-chat-takeout.test.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, it } from 'node:test' +import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types' + +import { detectFormat, parseFileSync } from '../index' + +interface FixtureMessage { + message_id: string + created_date?: string + updated_date?: string + creator: { + email?: string + name: string + user_type: string + } + text?: string + attached_files?: Array<{ + original_name?: string + export_name?: string + }> + quoted_message_metadata?: { + creator?: { + email?: string + name?: string + user_type?: string + } + text?: string + } +} + +async function parseGoogleChatFixture(options: { + chatType?: 'private' | 'group' + chatName?: string + date: string + messages?: FixtureMessage[] +}) { + const dir = mkdtempSync(join(tmpdir(), 'chatlab-google-chat-parser-')) + try { + const manifestPath = join(dir, 'google-chat-import.json') + writeFileSync( + manifestPath, + JSON.stringify({ + format: 'chatlab-google-chat-takeout', + version: 1, + chatId: options.chatType === 'group' ? 'Groups/Space sample' : 'Groups/DM sample', + chatType: options.chatType ?? 'private', + chatName: options.chatName, + userInfoFile: 'user_info.json', + groupInfoFile: 'group_info.json', + messagesFile: 'messages.json', + }), + 'utf8' + ) + writeFileSync( + join(dir, 'user_info.json'), + JSON.stringify({ + user: { + email: 'Owner@Example.com', + name: 'Owner', + user_type: 'Human', + }, + }), + 'utf8' + ) + writeFileSync( + join(dir, 'group_info.json'), + JSON.stringify({ + name: options.chatType === 'group' ? options.chatName : undefined, + members: [ + { email: 'Owner@Example.com', name: 'Owner', user_type: 'Human' }, + { email: 'Other@Example.com', name: 'Other User', user_type: 'Human' }, + ], + }), + 'utf8' + ) + writeFileSync( + join(dir, 'messages.json'), + JSON.stringify({ + messages: options.messages ?? [ + { + message_id: 'message-1', + created_date: options.date, + creator: { + email: 'Other@Example.com', + name: 'Other User', + user_type: 'Human', + }, + text: 'Hello', + }, + { + message_id: 'message-2', + created_date: options.date, + creator: { + email: 'Owner@Example.com', + name: 'Owner', + user_type: 'Human', + }, + attached_files: [{ original_name: 'voice_message.m4a', export_name: 'File-voice_message.m4a' }], + }, + ], + }), + 'utf8' + ) + + assert.equal(detectFormat(manifestPath)?.id, 'google-chat-takeout') + return await parseFileSync(manifestPath) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +describe('google chat takeout parser', () => { + it('parses Chinese UTC dates and attachment-only messages', async () => { + const result = await parseGoogleChatFixture({ + date: '2026年5月29日星期五 UTC 03:00:29', + }) + + assert.deepEqual(result.meta, { + name: 'Other User', + platform: KNOWN_PLATFORMS.GOOGLE_CHAT, + type: ChatType.PRIVATE, + ownerId: 'owner@example.com', + }) + assert.deepEqual( + result.members.map((member) => member.platformId), + ['owner@example.com', 'other@example.com'] + ) + assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 0, 29) / 1000) + assert.equal(result.messages[0].platformMessageId, 'message-1') + assert.equal(result.messages[1].type, MessageType.FILE) + assert.equal(result.messages[1].content, '[附件] voice_message.m4a') + }) + + it('parses English UTC dates with narrow no-break spaces', async () => { + const result = await parseGoogleChatFixture({ + date: 'Friday, May 29, 2026 at 3:00:29\u202fAM UTC', + }) + + assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 0, 29) / 1000) + }) + + it('preserves quoted text and attachment names alongside message text', async () => { + const date = 'Friday, May 29, 2026 at 3:00:29 AM UTC' + const result = await parseGoogleChatFixture({ + date, + messages: [ + { + message_id: 'message-3', + created_date: date, + creator: { + email: 'Other@Example.com', + name: 'Other User', + user_type: 'Human', + }, + text: 'Current reply', + attached_files: [ + { original_name: 'photo.jpg', export_name: 'File-photo.jpg' }, + { export_name: 'File-document.pdf' }, + ], + quoted_message_metadata: { + creator: { name: 'Owner' }, + text: 'Original text', + }, + }, + ], + }) + + assert.equal(result.messages[0].type, MessageType.TEXT) + assert.equal( + result.messages[0].content, + ['> Owner: Original text', 'Current reply', '[附件] photo.jpg', '[附件] File-document.pdf'].join('\n') + ) + }) + + it('uses Space metadata and marks invalid dates for importer diagnostics', async () => { + const result = await parseGoogleChatFixture({ + chatType: 'group', + chatName: 'Project Space', + date: 'Friday, February 31, 2026 at 3:00:29 AM UTC', + }) + + assert.equal(result.meta.name, 'Project Space') + assert.equal(result.meta.type, ChatType.GROUP) + assert.equal(result.meta.groupId, 'Groups/Space sample') + assert.equal(Number.isNaN(result.messages[0].timestamp), true) + }) + + it('falls back to updated_date for edited messages without created_date', async () => { + const result = await parseGoogleChatFixture({ + date: 'unused', + messages: [ + { + message_id: 'edited-message', + updated_date: 'Friday, May 29, 2026 at 3:24:57\u202fAM UTC', + creator: { + email: 'Other@Example.com', + name: 'Other User', + user_type: 'Human', + }, + text: 'Edited text', + }, + ], + }) + + assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 24, 57) / 1000) + }) +}) diff --git a/packages/parser/src/formats/google-chat-takeout.ts b/packages/parser/src/formats/google-chat-takeout.ts new file mode 100644 index 00000000..f0b5c9aa --- /dev/null +++ b/packages/parser/src/formats/google-chat-takeout.ts @@ -0,0 +1,373 @@ +/** + * Google Chat Takeout 内部导入格式解析器。 + * + * ZIP 读取与会话选择由 node-runtime 负责;本解析器只读取其生成的固定 manifest + * 和同目录 JSON 文件,从而复用现有 streaming importer。 + */ + +import * as fs from 'node:fs' +import * as path from 'node:path' +import streamChain from 'stream-chain' +import streamJson from 'stream-json' +import pickModule from 'stream-json/filters/Pick.js' +import streamValuesModule from 'stream-json/streamers/StreamValues.js' +import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types' +import type { + FormatFeature, + FormatModule, + ParseEvent, + ParseOptions, + ParsedMember, + ParsedMessage, + Parser, +} from '../types' +import { createProgress, getFileSize } from '../utils' + +const { chain } = streamChain +const { parser } = streamJson +const { pick } = pickModule +const { streamValues } = streamValuesModule + +interface GoogleChatManifest { + format: string + version: number + chatId: string + chatType: 'private' | 'group' + chatName?: string + userInfoFile: string + groupInfoFile: string + messagesFile: string +} + +interface GoogleChatUser { + email?: string + name?: string + user_type?: string +} + +interface GoogleChatUserInfo { + user?: GoogleChatUser +} + +interface GoogleChatGroupInfo { + name?: string + members?: GoogleChatUser[] +} + +interface GoogleChatAttachment { + original_name?: string + export_name?: string +} + +interface GoogleChatQuotedMessage { + creator?: GoogleChatUser + text?: string +} + +interface GoogleChatMessage { + message_id?: string + created_date?: string + updated_date?: string + creator?: GoogleChatUser + text?: string + attached_files?: GoogleChatAttachment[] + quoted_message_metadata?: GoogleChatQuotedMessage +} + +const ENGLISH_MONTHS = new Map([ + ['january', 0], + ['february', 1], + ['march', 2], + ['april', 3], + ['may', 4], + ['june', 5], + ['july', 6], + ['august', 7], + ['september', 8], + ['october', 9], + ['november', 10], + ['december', 11], +]) + +export const feature: FormatFeature = { + id: 'google-chat-takeout', + name: 'Google Chat Takeout', + platform: KNOWN_PLATFORMS.GOOGLE_CHAT, + priority: 24, + extensions: ['.json'], + signatures: { + head: [/"format"\s*:\s*"chatlab-google-chat-takeout"/], + requiredFields: ['format', 'version', 'messagesFile'], + }, +} + +function normalizeSpaces(value: string): string { + return value + .replace(/[\u00a0\u202f]/g, ' ') + .replace(/\s+/g, ' ') + .trim() +} + +function createUtcTimestamp( + year: number, + monthIndex: number, + day: number, + hour: number, + minute: number, + second: number +): number | null { + const timestampMs = Date.UTC(year, monthIndex, day, hour, minute, second) + const date = new Date(timestampMs) + if ( + date.getUTCFullYear() !== year || + date.getUTCMonth() !== monthIndex || + date.getUTCDate() !== day || + date.getUTCHours() !== hour || + date.getUTCMinutes() !== minute || + date.getUTCSeconds() !== second + ) { + return null + } + return Math.floor(timestampMs / 1000) +} + +function parseChineseUtcDate(value: string): number | null { + const match = normalizeSpaces(value).match( + /^(\d{4})年(\d{1,2})月(\d{1,2})日(?:星期[一二三四五六日天])?\s+UTC\s+(\d{1,2}):(\d{2}):(\d{2})$/ + ) + if (!match) return null + return createUtcTimestamp( + Number(match[1]), + Number(match[2]) - 1, + Number(match[3]), + Number(match[4]), + Number(match[5]), + Number(match[6]) + ) +} + +function parseEnglishUtcDate(value: string): number | null { + const match = normalizeSpaces(value).match( + /^(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),\s+([A-Za-z]+)\s+(\d{1,2}),\s+(\d{4})\s+at\s+(\d{1,2}):(\d{2}):(\d{2})\s+(AM|PM)\s+UTC$/i + ) + if (!match) return null + + const monthIndex = ENGLISH_MONTHS.get(match[1].toLowerCase()) + if (monthIndex === undefined) return null + + let hour = Number(match[4]) + if (hour < 1 || hour > 12) return null + if (match[7].toUpperCase() === 'AM') { + if (hour === 12) hour = 0 + } else if (hour !== 12) { + hour += 12 + } + + return createUtcTimestamp(Number(match[3]), monthIndex, Number(match[2]), hour, Number(match[5]), Number(match[6])) +} + +/** + * Google Takeout 会根据导出语言生成本地化日期文本。这里只接受已验证的 + * 中文和英文 UTC 形式,避免宿主机 locale 或 Date.parse 产生不稳定结果。 + */ +export function parseGoogleChatDate(value: string): number | null { + return parseChineseUtcDate(value) ?? parseEnglishUtcDate(value) +} + +function normalizeIdentity(user: GoogleChatUser | undefined): { platformId: string; name: string } { + const email = user?.email?.trim().toLowerCase() + const name = user?.name?.trim() || email || 'Unknown' + return { + platformId: email || name.toLowerCase(), + name, + } +} + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T +} + +function resolveManifestFile(manifestDir: string, relativePath: string): string { + const resolved = path.resolve(manifestDir, relativePath) + const relative = path.relative(manifestDir, resolved) + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Google Chat manifest path escapes its directory: ${relativePath}`) + } + return resolved +} + +function readManifest(filePath: string): { + manifest: GoogleChatManifest + userInfoPath: string + groupInfoPath: string + messagesPath: string +} { + const manifest = readJson(filePath) + if (manifest.format !== 'chatlab-google-chat-takeout' || manifest.version !== 1) { + throw new Error('Invalid Google Chat import manifest') + } + if (manifest.chatType !== 'private' && manifest.chatType !== 'group') { + throw new Error('Invalid Google Chat chat type') + } + + const manifestDir = path.dirname(filePath) + return { + manifest, + userInfoPath: resolveManifestFile(manifestDir, manifest.userInfoFile), + groupInfoPath: resolveManifestFile(manifestDir, manifest.groupInfoFile), + messagesPath: resolveManifestFile(manifestDir, manifest.messagesFile), + } +} + +function getAttachmentName(attachment: GoogleChatAttachment): string | null { + return attachment.original_name?.trim() || attachment.export_name?.trim() || null +} + +function buildQuotePrefix(message: GoogleChatMessage): string | null { + const quote = message.quoted_message_metadata + const quoteText = quote?.text?.trim() + if (!quoteText) return null + const creator = quote?.creator?.name?.trim() || quote?.creator?.email?.trim() + return creator ? `> ${creator}: ${quoteText}` : `> ${quoteText}` +} + +function buildMessageContent(message: GoogleChatMessage): string { + const parts: string[] = [] + const quote = buildQuotePrefix(message) + if (quote) parts.push(quote) + + const text = message.text?.trim() + if (text) parts.push(text) + + for (const attachment of message.attached_files ?? []) { + const name = getAttachmentName(attachment) + if (name) parts.push(`[附件] ${name}`) + } + + return parts.length > 0 ? parts.join('\n') : '[不支持的 Google Chat 消息]' +} + +function detectMessageType(message: GoogleChatMessage): MessageType { + if (message.text?.trim()) return MessageType.TEXT + if ((message.attached_files ?? []).some((attachment) => getAttachmentName(attachment))) { + return MessageType.FILE + } + return MessageType.OTHER +} + +function deriveChatName( + manifest: GoogleChatManifest, + groupInfo: GoogleChatGroupInfo, + ownerId: string | undefined +): string { + if (manifest.chatName?.trim()) return manifest.chatName.trim() + if (manifest.chatType === 'group' && groupInfo.name?.trim()) return groupInfo.name.trim() + + const otherMember = (groupInfo.members ?? []) + .map(normalizeIdentity) + .find((member) => !ownerId || member.platformId !== ownerId) + return ( + otherMember?.name || + groupInfo.members?.map((member) => normalizeIdentity(member).name).join(', ') || + manifest.chatId + ) +} + +async function* parseGoogleChat(options: ParseOptions): AsyncGenerator { + const { filePath, batchSize = 5000, onProgress, onLog } = options + const { manifest, userInfoPath, groupInfoPath, messagesPath } = readManifest(filePath) + const userInfo = readJson(userInfoPath) + const groupInfo = readJson(groupInfoPath) + const owner = userInfo.user ? normalizeIdentity(userInfo.user) : null + const totalBytes = getFileSize(messagesPath) + + const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') + yield { type: 'progress', data: initialProgress } + onProgress?.(initialProgress) + + yield { + type: 'meta', + data: { + name: deriveChatName(manifest, groupInfo, owner?.platformId), + platform: KNOWN_PLATFORMS.GOOGLE_CHAT, + type: manifest.chatType === 'group' ? ChatType.GROUP : ChatType.PRIVATE, + ...(manifest.chatType === 'group' ? { groupId: manifest.chatId } : {}), + ownerId: owner?.platformId, + }, + } + + const memberMap = new Map() + for (const member of groupInfo.members ?? []) { + const identity = normalizeIdentity(member) + memberMap.set(identity.platformId, { + platformId: identity.platformId, + accountName: identity.name, + }) + } + if (owner && !memberMap.has(owner.platformId)) { + memberMap.set(owner.platformId, { + platformId: owner.platformId, + accountName: owner.name, + }) + } + yield { type: 'members', data: Array.from(memberMap.values()) } + + const readStream = fs.createReadStream(messagesPath, { encoding: 'utf8' }) + let bytesRead = 0 + let messagesProcessed = 0 + readStream.on('data', (chunk: string | Buffer) => { + bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length + }) + + const pipeline = chain([readStream, parser(), pick({ filter: /^messages\.\d+$/ }), streamValues()]) + const messageBatch: ParsedMessage[] = [] + + try { + for await (const item of pipeline as AsyncIterable<{ value: GoogleChatMessage }>) { + const message = item.value + const sender = normalizeIdentity(message.creator) + messageBatch.push({ + platformMessageId: message.message_id ? String(message.message_id) : undefined, + senderPlatformId: sender.platformId, + senderAccountName: sender.name, + timestamp: parseGoogleChatDate(message.created_date ?? message.updated_date ?? '') ?? Number.NaN, + type: detectMessageType(message), + content: buildMessageContent(message), + }) + messagesProcessed++ + + if (messageBatch.length >= batchSize) { + yield { type: 'messages', data: messageBatch.splice(0) } + const progress = createProgress('parsing', bytesRead, totalBytes, messagesProcessed, '') + yield { type: 'progress', data: progress } + onProgress?.(progress) + } + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + onLog?.('error', `Failed to parse Google Chat messages: ${message}`) + yield { type: 'error', data: new Error(`Failed to parse Google Chat messages: ${message}`) } + return + } + + if (messageBatch.length > 0) { + yield { type: 'messages', data: messageBatch } + } + + const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') + yield { type: 'progress', data: doneProgress } + onProgress?.(doneProgress) + yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: memberMap.size } } +} + +export const parser_: Parser = { + feature, + parse: parseGoogleChat, +} + +const module_: FormatModule = { + feature, + parser: parser_, +} + +export default module_ diff --git a/packages/parser/src/formats/index.ts b/packages/parser/src/formats/index.ts index dc7522b4..e87d7f13 100644 --- a/packages/parser/src/formats/index.ts +++ b/packages/parser/src/formats/index.ts @@ -18,6 +18,7 @@ import telegramNativeSingle from './telegram-native-single' import whatsappNativeTxt from './whatsapp-native-txt' import qqNativeTxt from './qq-native-txt' import instagramNative from './instagram-native' +import googleChatTakeout from './google-chat-takeout' import lineNativeTxt from './line-native-txt' /** @@ -32,6 +33,7 @@ export const formats: FormatModule[] = [ tyrrrzDiscordExporter, // 优先级 20 - Tyrrrz/DiscordChatExporter telegramNative, // 优先级 22 - Telegram 官方全量导出 JSON telegramNativeSingle, // 优先级 23 - Telegram 单聊天导出 JSON + googleChatTakeout, // 优先级 24 - Google Chat Takeout 内部 manifest instagramNative, // 优先级 25 - Instagram 官方导出 whatsappNativeTxt, // 优先级 26 - WhatsApp 官方导出 TXT qqNativeTxt, // 优先级 30 - QQ 官方导出 TXT @@ -51,6 +53,7 @@ export { tyrrrzDiscordExporter, telegramNative, telegramNativeSingle, + googleChatTakeout, instagramNative, whatsappNativeTxt, qqNativeTxt, diff --git a/packages/shared-types/index.ts b/packages/shared-types/index.ts index d9cf517f..f93f5468 100644 --- a/packages/shared-types/index.ts +++ b/packages/shared-types/index.ts @@ -65,6 +65,7 @@ export const KNOWN_PLATFORMS = { WHATSAPP: 'whatsapp', TELEGRAM: 'telegram', INSTAGRAM: 'instagram', + GOOGLE_CHAT: 'google-chat', LINE: 'line', UNKNOWN: 'unknown', } as const diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f395edda..d5a4eb9e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,6 +340,9 @@ importers: sqlite-vec: specifier: ^0.1.9 version: 0.1.9 + yauzl: + specifier: ^3.4.0 + version: 3.4.0 optionalDependencies: '@node-rs/jieba': specifier: ^2.0.1 @@ -348,6 +351,9 @@ importers: '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 + '@types/yauzl': + specifier: ^3.4.0 + version: 3.4.0 packages/parser: dependencies: @@ -2378,6 +2384,9 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@types/yauzl@3.4.0': + resolution: {integrity: sha512-NRPn5w6h8dhcnmx3YIRQcqMywY/+nND/uOkJessedcrowO3C0AssHp3tMJpxKAwOhFOo0OV1y9VtsC5hbKKBAw==} + '@typescript-eslint/eslint-plugin@8.55.0': resolution: {integrity: sha512-1y/MVSz0NglV1ijHC8OT49mPJ4qhPYjiK08YUQVbIOyu+5k862LKUHFkpKHWu//zmr7hDR2rhwUm6gnCGNmGBQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -6024,6 +6033,10 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yauzl@3.4.0: + resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==} + engines: {node: '>=12'} + yjs@13.6.29: resolution: {integrity: sha512-kHqDPdltoXH+X4w1lVmMtddE3Oeqq48nM40FD5ojTd8xYhQpzIDcfE2keMSU5bAgRPJBe225WTUdyUgj1DtbiQ==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -8261,6 +8274,10 @@ snapshots: '@types/node': 25.2.2 optional: true + '@types/yauzl@3.4.0': + dependencies: + '@types/node': 25.2.2 + '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -12191,6 +12208,10 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 + yauzl@3.4.0: + dependencies: + pend: 1.2.0 + yjs@13.6.29: dependencies: lib0: 0.2.117