feat(parser): parse Google Chat takeout conversations

Add streaming zip reader dependency (fflate) and a new parser for Google
Chat takeout conversation files. Handles localized message types, reactions,
topic threads, and deduplication of multi-file exports.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

packages/node-runtime/package.json

@@ -17,12 +17,14 @@
     "onnxruntime-common": "1.24.3",
     "onnxruntime-node": "1.24.3",
     "sharp": "0.34.5",
-    "sqlite-vec": "^0.1.9"
+    "sqlite-vec": "^0.1.9",
+    "yauzl": "^3.4.0"
   },
   "optionalDependencies": {
     "@node-rs/jieba": "^2.0.1"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.13"
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/yauzl": "^3.4.0"
   }
 }

packages/parser/src/formats/google-chat-takeout.test.ts

@@ -0,0 +1,210 @@
+import assert from 'node:assert/strict'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { describe, it } from 'node:test'
+import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
+
+import { detectFormat, parseFileSync } from '../index'
+
+interface FixtureMessage {
+  message_id: string
+  created_date?: string
+  updated_date?: string
+  creator: {
+    email?: string
+    name: string
+    user_type: string
+  }
+  text?: string
+  attached_files?: Array<{
+    original_name?: string
+    export_name?: string
+  }>
+  quoted_message_metadata?: {
+    creator?: {
+      email?: string
+      name?: string
+      user_type?: string
+    }
+    text?: string
+  }
+}
+
+async function parseGoogleChatFixture(options: {
+  chatType?: 'private' | 'group'
+  chatName?: string
+  date: string
+  messages?: FixtureMessage[]
+}) {
+  const dir = mkdtempSync(join(tmpdir(), 'chatlab-google-chat-parser-'))
+  try {
+    const manifestPath = join(dir, 'google-chat-import.json')
+    writeFileSync(
+      manifestPath,
+      JSON.stringify({
+        format: 'chatlab-google-chat-takeout',
+        version: 1,
+        chatId: options.chatType === 'group' ? 'Groups/Space sample' : 'Groups/DM sample',
+        chatType: options.chatType ?? 'private',
+        chatName: options.chatName,
+        userInfoFile: 'user_info.json',
+        groupInfoFile: 'group_info.json',
+        messagesFile: 'messages.json',
+      }),
+      'utf8'
+    )
+    writeFileSync(
+      join(dir, 'user_info.json'),
+      JSON.stringify({
+        user: {
+          email: 'Owner@Example.com',
+          name: 'Owner',
+          user_type: 'Human',
+        },
+      }),
+      'utf8'
+    )
+    writeFileSync(
+      join(dir, 'group_info.json'),
+      JSON.stringify({
+        name: options.chatType === 'group' ? options.chatName : undefined,
+        members: [
+          { email: 'Owner@Example.com', name: 'Owner', user_type: 'Human' },
+          { email: 'Other@Example.com', name: 'Other User', user_type: 'Human' },
+        ],
+      }),
+      'utf8'
+    )
+    writeFileSync(
+      join(dir, 'messages.json'),
+      JSON.stringify({
+        messages: options.messages ?? [
+          {
+            message_id: 'message-1',
+            created_date: options.date,
+            creator: {
+              email: 'Other@Example.com',
+              name: 'Other User',
+              user_type: 'Human',
+            },
+            text: 'Hello',
+          },
+          {
+            message_id: 'message-2',
+            created_date: options.date,
+            creator: {
+              email: 'Owner@Example.com',
+              name: 'Owner',
+              user_type: 'Human',
+            },
+            attached_files: [{ original_name: 'voice_message.m4a', export_name: 'File-voice_message.m4a' }],
+          },
+        ],
+      }),
+      'utf8'
+    )
+
+    assert.equal(detectFormat(manifestPath)?.id, 'google-chat-takeout')
+    return await parseFileSync(manifestPath)
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+describe('google chat takeout parser', () => {
+  it('parses Chinese UTC dates and attachment-only messages', async () => {
+    const result = await parseGoogleChatFixture({
+      date: '2026年5月29日星期五 UTC 03:00:29',
+    })
+
+    assert.deepEqual(result.meta, {
+      name: 'Other User',
+      platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
+      type: ChatType.PRIVATE,
+      ownerId: 'owner@example.com',
+    })
+    assert.deepEqual(
+      result.members.map((member) => member.platformId),
+      ['owner@example.com', 'other@example.com']
+    )
+    assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 0, 29) / 1000)
+    assert.equal(result.messages[0].platformMessageId, 'message-1')
+    assert.equal(result.messages[1].type, MessageType.FILE)
+    assert.equal(result.messages[1].content, '[附件] voice_message.m4a')
+  })
+
+  it('parses English UTC dates with narrow no-break spaces', async () => {
+    const result = await parseGoogleChatFixture({
+      date: 'Friday, May 29, 2026 at 3:00:29\u202fAM UTC',
+    })
+
+    assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 0, 29) / 1000)
+  })
+
+  it('preserves quoted text and attachment names alongside message text', async () => {
+    const date = 'Friday, May 29, 2026 at 3:00:29 AM UTC'
+    const result = await parseGoogleChatFixture({
+      date,
+      messages: [
+        {
+          message_id: 'message-3',
+          created_date: date,
+          creator: {
+            email: 'Other@Example.com',
+            name: 'Other User',
+            user_type: 'Human',
+          },
+          text: 'Current reply',
+          attached_files: [
+            { original_name: 'photo.jpg', export_name: 'File-photo.jpg' },
+            { export_name: 'File-document.pdf' },
+          ],
+          quoted_message_metadata: {
+            creator: { name: 'Owner' },
+            text: 'Original text',
+          },
+        },
+      ],
+    })
+
+    assert.equal(result.messages[0].type, MessageType.TEXT)
+    assert.equal(
+      result.messages[0].content,
+      ['> Owner: Original text', 'Current reply', '[附件] photo.jpg', '[附件] File-document.pdf'].join('\n')
+    )
+  })
+
+  it('uses Space metadata and marks invalid dates for importer diagnostics', async () => {
+    const result = await parseGoogleChatFixture({
+      chatType: 'group',
+      chatName: 'Project Space',
+      date: 'Friday, February 31, 2026 at 3:00:29 AM UTC',
+    })
+
+    assert.equal(result.meta.name, 'Project Space')
+    assert.equal(result.meta.type, ChatType.GROUP)
+    assert.equal(result.meta.groupId, 'Groups/Space sample')
+    assert.equal(Number.isNaN(result.messages[0].timestamp), true)
+  })
+
+  it('falls back to updated_date for edited messages without created_date', async () => {
+    const result = await parseGoogleChatFixture({
+      date: 'unused',
+      messages: [
+        {
+          message_id: 'edited-message',
+          updated_date: 'Friday, May 29, 2026 at 3:24:57\u202fAM UTC',
+          creator: {
+            email: 'Other@Example.com',
+            name: 'Other User',
+            user_type: 'Human',
+          },
+          text: 'Edited text',
+        },
+      ],
+    })
+
+    assert.equal(result.messages[0].timestamp, Date.UTC(2026, 4, 29, 3, 24, 57) / 1000)
+  })
+})

packages/parser/src/formats/google-chat-takeout.ts

@@ -0,0 +1,373 @@
+/**
+ * Google Chat Takeout 内部导入格式解析器。
+ *
+ * ZIP 读取与会话选择由 node-runtime 负责；本解析器只读取其生成的固定 manifest
+ * 和同目录 JSON 文件，从而复用现有 streaming importer。
+ */
+
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import streamChain from 'stream-chain'
+import streamJson from 'stream-json'
+import pickModule from 'stream-json/filters/Pick.js'
+import streamValuesModule from 'stream-json/streamers/StreamValues.js'
+import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
+import type {
+  FormatFeature,
+  FormatModule,
+  ParseEvent,
+  ParseOptions,
+  ParsedMember,
+  ParsedMessage,
+  Parser,
+} from '../types'
+import { createProgress, getFileSize } from '../utils'
+
+const { chain } = streamChain
+const { parser } = streamJson
+const { pick } = pickModule
+const { streamValues } = streamValuesModule
+
+interface GoogleChatManifest {
+  format: string
+  version: number
+  chatId: string
+  chatType: 'private' | 'group'
+  chatName?: string
+  userInfoFile: string
+  groupInfoFile: string
+  messagesFile: string
+}
+
+interface GoogleChatUser {
+  email?: string
+  name?: string
+  user_type?: string
+}
+
+interface GoogleChatUserInfo {
+  user?: GoogleChatUser
+}
+
+interface GoogleChatGroupInfo {
+  name?: string
+  members?: GoogleChatUser[]
+}
+
+interface GoogleChatAttachment {
+  original_name?: string
+  export_name?: string
+}
+
+interface GoogleChatQuotedMessage {
+  creator?: GoogleChatUser
+  text?: string
+}
+
+interface GoogleChatMessage {
+  message_id?: string
+  created_date?: string
+  updated_date?: string
+  creator?: GoogleChatUser
+  text?: string
+  attached_files?: GoogleChatAttachment[]
+  quoted_message_metadata?: GoogleChatQuotedMessage
+}
+
+const ENGLISH_MONTHS = new Map([
+  ['january', 0],
+  ['february', 1],
+  ['march', 2],
+  ['april', 3],
+  ['may', 4],
+  ['june', 5],
+  ['july', 6],
+  ['august', 7],
+  ['september', 8],
+  ['october', 9],
+  ['november', 10],
+  ['december', 11],
+])
+
+export const feature: FormatFeature = {
+  id: 'google-chat-takeout',
+  name: 'Google Chat Takeout',
+  platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
+  priority: 24,
+  extensions: ['.json'],
+  signatures: {
+    head: [/"format"\s*:\s*"chatlab-google-chat-takeout"/],
+    requiredFields: ['format', 'version', 'messagesFile'],
+  },
+}
+
+function normalizeSpaces(value: string): string {
+  return value
+    .replace(/[\u00a0\u202f]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+function createUtcTimestamp(
+  year: number,
+  monthIndex: number,
+  day: number,
+  hour: number,
+  minute: number,
+  second: number
+): number | null {
+  const timestampMs = Date.UTC(year, monthIndex, day, hour, minute, second)
+  const date = new Date(timestampMs)
+  if (
+    date.getUTCFullYear() !== year ||
+    date.getUTCMonth() !== monthIndex ||
+    date.getUTCDate() !== day ||
+    date.getUTCHours() !== hour ||
+    date.getUTCMinutes() !== minute ||
+    date.getUTCSeconds() !== second
+  ) {
+    return null
+  }
+  return Math.floor(timestampMs / 1000)
+}
+
+function parseChineseUtcDate(value: string): number | null {
+  const match = normalizeSpaces(value).match(
+    /^(\d{4})年(\d{1,2})月(\d{1,2})日(?:星期[一二三四五六日天])?\s+UTC\s+(\d{1,2}):(\d{2}):(\d{2})$/
+  )
+  if (!match) return null
+  return createUtcTimestamp(
+    Number(match[1]),
+    Number(match[2]) - 1,
+    Number(match[3]),
+    Number(match[4]),
+    Number(match[5]),
+    Number(match[6])
+  )
+}
+
+function parseEnglishUtcDate(value: string): number | null {
+  const match = normalizeSpaces(value).match(
+    /^(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),\s+([A-Za-z]+)\s+(\d{1,2}),\s+(\d{4})\s+at\s+(\d{1,2}):(\d{2}):(\d{2})\s+(AM|PM)\s+UTC$/i
+  )
+  if (!match) return null
+
+  const monthIndex = ENGLISH_MONTHS.get(match[1].toLowerCase())
+  if (monthIndex === undefined) return null
+
+  let hour = Number(match[4])
+  if (hour < 1 || hour > 12) return null
+  if (match[7].toUpperCase() === 'AM') {
+    if (hour === 12) hour = 0
+  } else if (hour !== 12) {
+    hour += 12
+  }
+
+  return createUtcTimestamp(Number(match[3]), monthIndex, Number(match[2]), hour, Number(match[5]), Number(match[6]))
+}
+
+/**
+ * Google Takeout 会根据导出语言生成本地化日期文本。这里只接受已验证的
+ * 中文和英文 UTC 形式，避免宿主机 locale 或 Date.parse 产生不稳定结果。
+ */
+export function parseGoogleChatDate(value: string): number | null {
+  return parseChineseUtcDate(value) ?? parseEnglishUtcDate(value)
+}
+
+function normalizeIdentity(user: GoogleChatUser | undefined): { platformId: string; name: string } {
+  const email = user?.email?.trim().toLowerCase()
+  const name = user?.name?.trim() || email || 'Unknown'
+  return {
+    platformId: email || name.toLowerCase(),
+    name,
+  }
+}
+
+function readJson<T>(filePath: string): T {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T
+}
+
+function resolveManifestFile(manifestDir: string, relativePath: string): string {
+  const resolved = path.resolve(manifestDir, relativePath)
+  const relative = path.relative(manifestDir, resolved)
+  if (relative.startsWith('..') || path.isAbsolute(relative)) {
+    throw new Error(`Google Chat manifest path escapes its directory: ${relativePath}`)
+  }
+  return resolved
+}
+
+function readManifest(filePath: string): {
+  manifest: GoogleChatManifest
+  userInfoPath: string
+  groupInfoPath: string
+  messagesPath: string
+} {
+  const manifest = readJson<GoogleChatManifest>(filePath)
+  if (manifest.format !== 'chatlab-google-chat-takeout' || manifest.version !== 1) {
+    throw new Error('Invalid Google Chat import manifest')
+  }
+  if (manifest.chatType !== 'private' && manifest.chatType !== 'group') {
+    throw new Error('Invalid Google Chat chat type')
+  }
+
+  const manifestDir = path.dirname(filePath)
+  return {
+    manifest,
+    userInfoPath: resolveManifestFile(manifestDir, manifest.userInfoFile),
+    groupInfoPath: resolveManifestFile(manifestDir, manifest.groupInfoFile),
+    messagesPath: resolveManifestFile(manifestDir, manifest.messagesFile),
+  }
+}
+
+function getAttachmentName(attachment: GoogleChatAttachment): string | null {
+  return attachment.original_name?.trim() || attachment.export_name?.trim() || null
+}
+
+function buildQuotePrefix(message: GoogleChatMessage): string | null {
+  const quote = message.quoted_message_metadata
+  const quoteText = quote?.text?.trim()
+  if (!quoteText) return null
+  const creator = quote?.creator?.name?.trim() || quote?.creator?.email?.trim()
+  return creator ? `> ${creator}: ${quoteText}` : `> ${quoteText}`
+}
+
+function buildMessageContent(message: GoogleChatMessage): string {
+  const parts: string[] = []
+  const quote = buildQuotePrefix(message)
+  if (quote) parts.push(quote)
+
+  const text = message.text?.trim()
+  if (text) parts.push(text)
+
+  for (const attachment of message.attached_files ?? []) {
+    const name = getAttachmentName(attachment)
+    if (name) parts.push(`[附件] ${name}`)
+  }
+
+  return parts.length > 0 ? parts.join('\n') : '[不支持的 Google Chat 消息]'
+}
+
+function detectMessageType(message: GoogleChatMessage): MessageType {
+  if (message.text?.trim()) return MessageType.TEXT
+  if ((message.attached_files ?? []).some((attachment) => getAttachmentName(attachment))) {
+    return MessageType.FILE
+  }
+  return MessageType.OTHER
+}
+
+function deriveChatName(
+  manifest: GoogleChatManifest,
+  groupInfo: GoogleChatGroupInfo,
+  ownerId: string | undefined
+): string {
+  if (manifest.chatName?.trim()) return manifest.chatName.trim()
+  if (manifest.chatType === 'group' && groupInfo.name?.trim()) return groupInfo.name.trim()
+
+  const otherMember = (groupInfo.members ?? [])
+    .map(normalizeIdentity)
+    .find((member) => !ownerId || member.platformId !== ownerId)
+  return (
+    otherMember?.name ||
+    groupInfo.members?.map((member) => normalizeIdentity(member).name).join(', ') ||
+    manifest.chatId
+  )
+}
+
+async function* parseGoogleChat(options: ParseOptions): AsyncGenerator<ParseEvent, void, unknown> {
+  const { filePath, batchSize = 5000, onProgress, onLog } = options
+  const { manifest, userInfoPath, groupInfoPath, messagesPath } = readManifest(filePath)
+  const userInfo = readJson<GoogleChatUserInfo>(userInfoPath)
+  const groupInfo = readJson<GoogleChatGroupInfo>(groupInfoPath)
+  const owner = userInfo.user ? normalizeIdentity(userInfo.user) : null
+  const totalBytes = getFileSize(messagesPath)
+
+  const initialProgress = createProgress('parsing', 0, totalBytes, 0, '')
+  yield { type: 'progress', data: initialProgress }
+  onProgress?.(initialProgress)
+
+  yield {
+    type: 'meta',
+    data: {
+      name: deriveChatName(manifest, groupInfo, owner?.platformId),
+      platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
+      type: manifest.chatType === 'group' ? ChatType.GROUP : ChatType.PRIVATE,
+      ...(manifest.chatType === 'group' ? { groupId: manifest.chatId } : {}),
+      ownerId: owner?.platformId,
+    },
+  }
+
+  const memberMap = new Map<string, ParsedMember>()
+  for (const member of groupInfo.members ?? []) {
+    const identity = normalizeIdentity(member)
+    memberMap.set(identity.platformId, {
+      platformId: identity.platformId,
+      accountName: identity.name,
+    })
+  }
+  if (owner && !memberMap.has(owner.platformId)) {
+    memberMap.set(owner.platformId, {
+      platformId: owner.platformId,
+      accountName: owner.name,
+    })
+  }
+  yield { type: 'members', data: Array.from(memberMap.values()) }
+
+  const readStream = fs.createReadStream(messagesPath, { encoding: 'utf8' })
+  let bytesRead = 0
+  let messagesProcessed = 0
+  readStream.on('data', (chunk: string | Buffer) => {
+    bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
+  })
+
+  const pipeline = chain([readStream, parser(), pick({ filter: /^messages\.\d+$/ }), streamValues()])
+  const messageBatch: ParsedMessage[] = []
+
+  try {
+    for await (const item of pipeline as AsyncIterable<{ value: GoogleChatMessage }>) {
+      const message = item.value
+      const sender = normalizeIdentity(message.creator)
+      messageBatch.push({
+        platformMessageId: message.message_id ? String(message.message_id) : undefined,
+        senderPlatformId: sender.platformId,
+        senderAccountName: sender.name,
+        timestamp: parseGoogleChatDate(message.created_date ?? message.updated_date ?? '') ?? Number.NaN,
+        type: detectMessageType(message),
+        content: buildMessageContent(message),
+      })
+      messagesProcessed++
+
+      if (messageBatch.length >= batchSize) {
+        yield { type: 'messages', data: messageBatch.splice(0) }
+        const progress = createProgress('parsing', bytesRead, totalBytes, messagesProcessed, '')
+        yield { type: 'progress', data: progress }
+        onProgress?.(progress)
+      }
+    }
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    onLog?.('error', `Failed to parse Google Chat messages: ${message}`)
+    yield { type: 'error', data: new Error(`Failed to parse Google Chat messages: ${message}`) }
+    return
+  }
+
+  if (messageBatch.length > 0) {
+    yield { type: 'messages', data: messageBatch }
+  }
+
+  const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '')
+  yield { type: 'progress', data: doneProgress }
+  onProgress?.(doneProgress)
+  yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: memberMap.size } }
+}
+
+export const parser_: Parser = {
+  feature,
+  parse: parseGoogleChat,
+}
+
+const module_: FormatModule = {
+  feature,
+  parser: parser_,
+}
+
+export default module_

packages/parser/src/formats/index.ts

@@ -18,6 +18,7 @@ import telegramNativeSingle from './telegram-native-single'
 import whatsappNativeTxt from './whatsapp-native-txt'
 import qqNativeTxt from './qq-native-txt'
 import instagramNative from './instagram-native'
+import googleChatTakeout from './google-chat-takeout'
 import lineNativeTxt from './line-native-txt'
 
 /**
@@ -32,6 +33,7 @@ export const formats: FormatModule[] = [
   tyrrrzDiscordExporter, // 优先级 20 - Tyrrrz/DiscordChatExporter
   telegramNative, // 优先级 22 - Telegram 官方全量导出 JSON
   telegramNativeSingle, // 优先级 23 - Telegram 单聊天导出 JSON
+  googleChatTakeout, // 优先级 24 - Google Chat Takeout 内部 manifest
   instagramNative, // 优先级 25 - Instagram 官方导出
   whatsappNativeTxt, // 优先级 26 - WhatsApp 官方导出 TXT
   qqNativeTxt, // 优先级 30 - QQ 官方导出 TXT
@@ -51,6 +53,7 @@ export {
   tyrrrzDiscordExporter,
   telegramNative,
   telegramNativeSingle,
+  googleChatTakeout,
   instagramNative,
   whatsappNativeTxt,
   qqNativeTxt,

packages/shared-types/index.ts

@@ -65,6 +65,7 @@ export const KNOWN_PLATFORMS = {
   WHATSAPP: 'whatsapp',
   TELEGRAM: 'telegram',
   INSTAGRAM: 'instagram',
+  GOOGLE_CHAT: 'google-chat',
   LINE: 'line',
   UNKNOWN: 'unknown',
 } as const