perf(parser): optimize browser parsing hot paths

This commit is contained in:
digua committed 2026-07-30 22:15:13 +08:00
1 parent 4a1f0b54a1
commit f0bdba63fb
13 files changed
+299 -83

No files matched your search

+22 -1
View File
@@ -20,7 +20,7 @@ describe('LINE browser-safe text parser', () => {
'03:05pm\tBob\t[Photo]',
'03:06pm\t\tAlice joined the group',
'',
].join('\n')
].join('\r\n')
assert.equal(detectLineText(content, '[LINE] Project Team.txt'), true)
assert.equal(detectLineText('2024/01/02 03:04 - Alice: hello', '与Alice的 WhatsApp 聊天.txt'), false)
@@ -85,4 +85,25 @@ describe('LINE browser-safe text parser', () => {
assert.equal(result.messages.length, 1002)
assert.ok(checks > 10)
})
it('classifies prefixed media, links, locations, and system messages', async () => {
const content = [
'[LINE] Chat history in Project Team',
'Saved on: 2024/01/03 09:00',
'',
'2024.01.02 Tuesday',
'03:04\tAlice\t[File] report.pdf',
'03:05\tAlice\thttps://example.com',
'03:06\tAlice\t[null] https://maps.google.com/example',
'03:07\tAlice\tBob left the group',
'',
].join('\n')
const result = await parseLineText(content, '[LINE] Project Team.txt')
assert.deepEqual(
result.messages.map((message) => message.type),
[MessageType.FILE, MessageType.LINK, MessageType.LOCATION, MessageType.SYSTEM]
)
})
})
+8 -6
View File
@@ -75,6 +75,7 @@ const SPECIAL_MESSAGE_TYPES: Record<string, MessageType> = {
'[記事本]': MessageType.TEXT,
'[ノート]': MessageType.TEXT,
}
const SPECIAL_MESSAGE_TYPE_ENTRIES = Object.entries(SPECIAL_MESSAGE_TYPES)
export function detectLineText(head: string, fileName: string): boolean {
const basename = getBasename(fileName)
@@ -90,20 +91,21 @@ export async function parseLineText(
fileName: string,
options: LineTextParseOptions = {}
): Promise<LineTextParseResult> {
const lines = content.split('\n').map((line) => line.replace(/\r$/, ''))
const lines = content.split('\n')
const memberMap = new Map<string, ParsedMember>()
const messages: ParsedMessage[] = []
const header = lines.length > 0 ? extractNameFromHeader(lines[0].trim()) : null
const header = lines.length > 0 ? extractNameFromHeader(lines[0].replace(/\r$/, '').trim()) : null
const chatName = header?.name ?? extractNameFromFileName(fileName)
let currentDate: Date | null = null
let lastMessage: ParsedMessage | null = null
let quotedMultiline = false
let lineIndex = header ? 3 : 0
let useTabSeparator = Boolean(header)
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
if (!header) {
for (const line of lines) {
for (const rawLine of lines) {
const line = rawLine.replace(/\r$/, '')
if (TAB_MESSAGE_PATTERN.test(line)) {
useTabSeparator = true
break
@@ -117,7 +119,7 @@ export async function parseLineText(
for (; lineIndex < lines.length; lineIndex += 1) {
options.checkCancelled?.()
const line = lines[lineIndex]
const line = lines[lineIndex].replace(/\r$/, '')
const date = parseDateLine(line)
if (date) {
currentDate = date
@@ -262,7 +264,7 @@ function parseTime(timeText: string): { hours: number; minutes: number } {
}
function detectMessageType(content: string): MessageType {
for (const [pattern, type] of Object.entries(SPECIAL_MESSAGE_TYPES)) {
for (const [pattern, type] of SPECIAL_MESSAGE_TYPE_ENTRIES) {
if (content === pattern || content.startsWith(pattern)) return type
}
if (content.startsWith('[null]') && content.includes('maps.google.com')) return MessageType.LOCATION
+1 -1
View File
@@ -45,7 +45,7 @@ export async function parseQqText(
): Promise<QqTextParseResult> {
const accumulator = new QqTextAccumulator(fileName)
const lines = content.split('\n')
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
for (let index = 0; index < lines.length; index += 1) {
options.checkCancelled?.()
+2 -2
View File
@@ -65,7 +65,7 @@ export async function scanTelegramChatsJson(
options.checkCancelled?.()
const value = parseTelegramFullExport(content)
const chats: TelegramChatInfo[] = []
const yieldEvery = Math.max(1, options.yieldEvery ?? 100)
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
for (let index = 0; index < value.chats.list.length; index += 1) {
options.checkCancelled?.()
@@ -124,7 +124,7 @@ async function parseTelegramChat(
const memberMap = new Map<string, ParsedMember>()
const messages: ParsedMessage[] = []
const chatType = mapChatType(value.type)
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
for (let index = 0; index < value.messages.length; index += 1) {
options.checkCancelled?.()
@@ -0,0 +1,45 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
import { parseWeFlowJson } from './weflow'
describe('WeFlow browser-safe JSON parser', () => {
it('parses messages and only reports intermediate progress at cooperative yield points', async () => {
const progress: Array<{ progress: number; messagesProcessed: number }> = []
const content = JSON.stringify({
weflow: { version: '1.0.0' },
session: { wxid: 'group@chatroom', displayName: 'Project Team', type: '群聊' },
messages: Array.from({ length: 5 }, (_, index) => ({
localId: index + 1,
createTime: 1704164645 + index,
type: '文本消息',
content: `message ${index}`,
isSend: index === 0 ? 1 : 0,
senderUsername: `member-${index % 2}`,
senderDisplayName: `Member ${index % 2}`,
})),
})
const result = await parseWeFlowJson(content, 'project.json', {
yieldEvery: 2,
onProgress: (value) => progress.push(value),
})
assert.deepEqual(result.meta, {
name: 'Project Team',
platform: KNOWN_PLATFORMS.WECHAT,
type: ChatType.GROUP,
groupId: 'group@chatroom',
groupAvatar: undefined,
ownerId: 'member-0',
})
assert.equal(result.messages.length, 5)
assert.equal(result.messages[0]?.type, MessageType.TEXT)
assert.deepEqual(
progress.map((value) => value.messagesProcessed),
[2, 4, 5]
)
assert.equal(progress.at(-1)?.progress, 1)
})
})
+7 -14
View File
@@ -61,10 +61,16 @@ export async function parseWeFlowJson(
const messageValues = Array.isArray(value.messages) ? value.messages : []
const members = new Map<string, MemberState>()
const messages: ParsedMessage[] = []
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
let ownerId: string | undefined
for (let index = 0; index < messageValues.length; index += 1) {
await cooperate(index, options)
options.checkCancelled?.()
if (index > 0 && index % yieldEvery === 0) {
options.onProgress?.({ progress: index / messageValues.length, messagesProcessed: messages.length })
await new Promise<void>((resolve) => setTimeout(resolve, 0))
options.checkCancelled?.()
}
const message = messageValues[index]
if (!isRecord(message)) continue
@@ -92,10 +98,6 @@ export async function parseWeFlowJson(
type: convertMessageType(stringValue(message.type)),
content: normalizeContent(message.content),
})
options.onProgress?.({
progress: messageValues.length === 0 ? 1 : (index + 1) / messageValues.length,
messagesProcessed: messages.length,
})
}
options.checkCancelled?.()
@@ -183,12 +185,3 @@ function stringValue(value: unknown): string | undefined {
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
async function cooperate(index: number, options: WeFlowJsonParseOptions): Promise<void> {
options.checkCancelled?.()
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
if (index > 0 && index % yieldEvery === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 0))
options.checkCancelled?.()
}
}
+1 -1
View File
@@ -108,7 +108,7 @@ export async function parseWhatsAppText(
): Promise<WhatsAppTextParseResult> {
const accumulator = new WhatsAppTextAccumulator(fileName)
const lines = content.split('\n')
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
for (let index = 0; index < lines.length; index += 1) {
options.checkCancelled?.()
@@ -0,0 +1,62 @@
import assert from 'node:assert/strict'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, it } from 'node:test'
import { MessageType } from '@openchatlab/shared-types'
import { detectFormat, parseFileSync } from '../index'
function encodeInstagramText(value: string): string {
return Buffer.from(value, 'utf8').toString('latin1')
}
describe('Instagram native parser', () => {
it('decodes Instagram text and classifies system messages', async () => {
const directory = mkdtempSync(join(tmpdir(), 'chatlab-instagram-parser-'))
try {
const filePath = join(directory, 'message_1.json')
writeFileSync(
filePath,
JSON.stringify({
participants: [{ name: encodeInstagramText('小明') }, { name: encodeInstagramText('小红') }],
messages: [
{
sender_name: encodeInstagramText('小红'),
timestamp_ms: 2000,
content: 'Alice added Bob to the group',
is_geoblocked_for_viewer: false,
},
{
sender_name: encodeInstagramText('小明'),
timestamp_ms: 1000,
content: encodeInstagramText('你好'),
is_geoblocked_for_viewer: false,
},
],
title: encodeInstagramText('小红'),
thread_path: 'inbox/example',
}),
'utf8'
)
assert.equal(detectFormat(filePath)?.id, 'instagram-native')
const result = await parseFileSync(filePath)
assert.equal(result.meta.name, '小红')
assert.deepEqual(
result.messages.map((message) => ({
sender: message.senderAccountName,
type: message.type,
content: message.content,
})),
[
{ sender: '小明', type: MessageType.TEXT, content: '你好' },
{ sender: '小红', type: MessageType.SYSTEM, content: 'Alice added Bob to the group' },
]
)
} finally {
rmSync(directory, { recursive: true, force: true })
}
})
})
+24 -21
View File
@@ -106,6 +106,8 @@ interface InstagramData {
// ==================== 辅助函数 ====================
const UTF8_DECODER = new TextDecoder('utf-8')
/**
* 解码 Instagram 特殊编码的文本
* Instagram 将 UTF-8 字节按 Latin-1 编码后存储
@@ -118,7 +120,7 @@ function decodeInstagramText(text: string): string {
bytes[i] = text.charCodeAt(i)
}
// 用 UTF-8 解码
return new TextDecoder('utf-8').decode(bytes)
return UTF8_DECODER.decode(bytes)
} catch {
return text // 解码失败则返回原文
}
@@ -135,23 +137,24 @@ function extractNameFromFilePath(filePath: string): string {
/**
* 判断是否为系统消息
*/
const SYSTEM_MESSAGE_PATTERNS = [
'You created the group',
'created the group',
'added',
'to the group',
'left the group',
'removed',
'named the group',
'changed the group photo',
'Reacted',
'sent an attachment',
'liked a message',
'changed the theme',
'set the nickname',
]
function isSystemMessage(content: string): boolean {
const systemPatterns = [
'You created the group',
'created the group',
'added',
'to the group',
'left the group',
'removed',
'named the group',
'changed the group photo',
'Reacted',
'sent an attachment',
'liked a message',
'changed the theme',
'set the nickname',
]
return systemPatterns.some((p) => content.includes(p))
return SYSTEM_MESSAGE_PATTERNS.some((pattern) => content.includes(pattern))
}
/**
@@ -286,11 +289,11 @@ async function* parseInstagram(options: ParseOptions): AsyncGenerator<ParseEvent
const members = Array.from(memberMap.values())
yield { type: 'members', data: members }
// 处理消息(Instagram 消息是逆序的,需要反转)
const reversedMessages = [...data.messages].reverse()
// Instagram 消息是逆序的;从尾部遍历即可按时间正序输出,避免复制整份消息数组。
const messageBatch: ParsedMessage[] = []
for (const msg of reversedMessages) {
for (let index = data.messages.length - 1; index >= 0; index -= 1) {
const msg = data.messages[index]
const senderName = decodeInstagramText(msg.sender_name)
const timestamp = Math.floor(msg.timestamp_ms / 1000) // 毫秒转秒
const type = detectMessageType(msg)
@@ -321,7 +324,7 @@ async function* parseInstagram(options: ParseOptions): AsyncGenerator<ParseEvent
const progress = createProgress(
'parsing',
Math.floor((messagesProcessed / reversedMessages.length) * totalBytes),
Math.floor((messagesProcessed / data.messages.length) * totalBytes),
totalBytes,
messagesProcessed,
`已处理 ${messagesProcessed} 条消息...`
@@ -23,7 +23,16 @@ describe('ChatLab browser parser', () => {
meta: { name: 'Fixture', platform: 'wechat', type: 'group', ownerId: 'alice' },
members: [{ platformId: 'alice', accountName: 'Alice' }],
messages: [
{ sender: 'alice', accountName: 'Alice', timestamp: 1, type: 0, content: 'hello' },
{
sender: 'alice',
accountName: 'Alice',
groupNickname: 'A',
timestamp: 1,
type: 0,
content: 'hello',
platformMessageId: 'message-1',
replyToMessageId: 'message-0',
},
{ sender: 'bob', accountName: 'Bob', timestamp: 2, type: 0, content: null },
],
})
@@ -44,6 +53,16 @@ describe('ChatLab browser parser', () => {
['alice', 'bob']
)
assert.equal(parsed.messages.length, 2)
assert.deepEqual(parsed.messages[0], {
senderPlatformId: 'alice',
senderAccountName: 'Alice',
senderGroupNickname: 'A',
timestamp: 1,
type: 0,
content: 'hello',
platformMessageId: 'message-1',
replyToMessageId: 'message-0',
})
assert.equal(parsed.messages[1].senderPlatformId, 'bob')
})
@@ -114,4 +133,45 @@ describe('ChatLab browser parser', () => {
/cancelled/
)
})
it('throttles JSON and JSONL progress updates to cooperative yield points', async () => {
const messages = Array.from({ length: 5 }, (_, index) => ({
sender: `member-${index}`,
accountName: `Member ${index}`,
timestamp: index + 1,
type: 0,
content: 'message',
}))
const jsonProgress: number[] = []
await parseChatLabSource(
source(
'progress.json',
JSON.stringify({
chatlab: { version: '1', exportedAt: 1 },
meta: { name: 'Progress', platform: 'unknown', type: 'group' },
messages,
})
),
{ yieldEvery: 2, onProgress: (value) => jsonProgress.push(value.messagesProcessed) }
)
const jsonlProgress: number[] = []
await parseChatLabSource(
source(
'progress.jsonl',
[
JSON.stringify({
_type: 'header',
chatlab: { version: '1', exportedAt: 1 },
meta: { name: 'Progress', platform: 'unknown', type: 'group' },
}),
...messages.map((message) => JSON.stringify({ _type: 'message', ...message })),
].join('\n')
),
{ yieldEvery: 2, onProgress: (value) => jsonlProgress.push(value.messagesProcessed) }
)
assert.deepEqual(jsonProgress, [2, 4, 5])
assert.deepEqual(jsonlProgress, [0, 2, 4, 5])
})
})
@@ -116,17 +116,23 @@ async function parseJson(content: string, options: ParseChatLabSourceOptions): P
}
const messages: BrowserParsedMessage[] = []
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
for (let index = 0; index < root.messages.length; index += 1) {
await cooperate(index, options)
options.checkCancelled?.()
if (index > 0 && index % yieldEvery === 0) {
options.onProgress?.({
stage: 'parsing',
progress: index / root.messages.length,
messagesProcessed: messages.length,
})
await new Promise<void>((resolve) => setTimeout(resolve, 0))
options.checkCancelled?.()
}
messages.push(parseMessage(root.messages[index], `messages[${index}]`))
options.onProgress?.({
stage: 'parsing',
progress: root.messages.length === 0 ? 1 : (index + 1) / root.messages.length,
messagesProcessed: index + 1,
})
}
options.checkCancelled?.()
options.onProgress?.({ stage: 'parsing', progress: 1, messagesProcessed: messages.length })
return {
formatId: 'chatlab',
meta,
@@ -142,6 +148,7 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
let firstRecordSeen = false
let cursor = 0
let lineNumber = 0
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
while (cursor <= content.length) {
const nextBreak = content.indexOf('\n', cursor)
@@ -149,9 +156,18 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
const line = content.slice(cursor, end).replace(/\r$/, '').trim()
lineNumber += 1
cursor = nextBreak === -1 ? content.length + 1 : nextBreak + 1
options.checkCancelled?.()
if (lineNumber > 0 && lineNumber % yieldEvery === 0) {
options.onProgress?.({
stage: 'parsing',
progress: content.length === 0 ? 1 : Math.min(1, cursor / content.length),
messagesProcessed: messages.length,
})
await new Promise<void>((resolve) => setTimeout(resolve, 0))
options.checkCancelled?.()
}
if (!line || line.startsWith('#')) continue
await cooperate(lineNumber, options)
let value: unknown
try {
value = JSON.parse(line)
@@ -182,11 +198,6 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
break
case 'message':
messages.push(parseMessage(value, `line ${lineNumber}`))
options.onProgress?.({
stage: 'parsing',
progress: content.length === 0 ? 1 : Math.min(1, cursor / content.length),
messagesProcessed: messages.length,
})
break
default:
throw new WebRuntimeError('INVALID_IMPORT_FILE', `Unknown ChatLab JSONL record type at line ${lineNumber}`)
@@ -244,16 +255,20 @@ function parseMessage(value: unknown, path: string): BrowserParsedMessage {
const content = value.content
if (content !== null && typeof content !== 'string') throw invalidField(`${path}.content`, 'must be a string or null')
return omitUndefined({
const message: BrowserParsedMessage = {
senderPlatformId: sender,
senderAccountName: optionalString(value.accountName, `${path}.accountName`) ?? sender,
senderGroupNickname: optionalString(value.groupNickname, `${path}.groupNickname`),
timestamp,
type,
content,
platformMessageId: optionalString(value.platformMessageId, `${path}.platformMessageId`),
replyToMessageId: optionalString(value.replyToMessageId, `${path}.replyToMessageId`),
})
}
const senderGroupNickname = optionalString(value.groupNickname, `${path}.groupNickname`)
const platformMessageId = optionalString(value.platformMessageId, `${path}.platformMessageId`)
const replyToMessageId = optionalString(value.replyToMessageId, `${path}.replyToMessageId`)
if (senderGroupNickname !== undefined) message.senderGroupNickname = senderGroupNickname
if (platformMessageId !== undefined) message.platformMessageId = platformMessageId
if (replyToMessageId !== undefined) message.replyToMessageId = replyToMessageId
return message
}
function mergeInferredMembers(
@@ -272,15 +287,6 @@ function mergeInferredMembers(
return [...byId.values()]
}
async function cooperate(index: number, options: ParseChatLabSourceOptions): Promise<void> {
options.checkCancelled?.()
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
if (index > 0 && index % yieldEvery === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 0))
options.checkCancelled?.()
}
}
function firstMeaningfulLine(content: string): string | undefined {
return content
.split(/\r?\n/)
@@ -34,9 +34,12 @@ describe('browser Rust WASM parser', () => {
{
sender: 'alice',
accountName: 'Alice',
groupNickname: 'A',
timestamp: 1,
type: 0,
content: 'hello chatlab',
platformMessageId: 'message-1',
replyToMessageId: 'message-0',
},
],
})
@@ -54,6 +57,16 @@ describe('browser Rust WASM parser', () => {
chatlab.messages.map((message) => message.content),
['hello chatlab']
)
assert.deepEqual(chatlab.messages[0], {
senderPlatformId: 'alice',
senderAccountName: 'Alice',
senderGroupNickname: 'A',
timestamp: 1,
type: 0,
content: 'hello chatlab',
platformMessageId: 'message-1',
replyToMessageId: 'message-0',
})
assert.deepEqual(chatlab, chatlabTs)
const weflowSource = source('weflow.json', {
+21 -10
View File
@@ -99,7 +99,7 @@ export async function parseWithWasm(
const batchJson = parser.take_batch_json(batchSize)
if (batchJson === undefined) break
const batch = JSON.parse(batchJson) as WasmNativeMessage[]
messages.push(...batch.map((message) => mapMessage(message, formatId)))
for (const message of batch) messages.push(mapMessage(message, formatId))
options.onProgress?.({
stage: 'parsing',
progress: summary.messageCount === 0 ? 1 : messages.length / summary.messageCount,
@@ -218,21 +218,32 @@ function mapMembers(
}
function mapMessage(message: WasmNativeMessage, formatId: BrowserWasmFormatId): BrowserParsedMessage {
const mapped = {
platformMessageId: optionalString(message.platformMessageId),
const platformMessageId = optionalString(message.platformMessageId)
const senderGroupNickname = optionalString(message.senderGroupNickname)
const replyToMessageId = optionalString(message.replyToMessageId)
if (formatId === 'weflow') {
return {
platformMessageId,
senderPlatformId: message.senderPlatformId,
senderAccountName: message.senderAccountName,
senderGroupNickname,
timestamp: message.timestamp as number,
type: message.messageType,
content: message.content ?? null,
}
}
const mapped: BrowserParsedMessage = {
senderPlatformId: message.senderPlatformId,
senderAccountName: message.senderAccountName,
senderGroupNickname: optionalString(message.senderGroupNickname),
timestamp: message.timestamp as number,
type: message.messageType,
content: message.content ?? null,
replyToMessageId: optionalString(message.replyToMessageId),
}
if (formatId === 'weflow') {
const { replyToMessageId: _replyToMessageId, ...weflowMessage } = mapped
return weflowMessage
}
return compact(mapped)
if (platformMessageId !== undefined) mapped.platformMessageId = platformMessageId
if (senderGroupNickname !== undefined) mapped.senderGroupNickname = senderGroupNickname
if (replyToMessageId !== undefined) mapped.replyToMessageId = replyToMessageId
return mapped
}
function optionalString(value: unknown): string | undefined {