mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
perf(parser): optimize browser parsing hot paths
This commit is contained in:
1 parent
4a1f0b54a1
commit
f0bdba63fb
13 files changed
+299
-83
No files matched your search
@@ -20,7 +20,7 @@ describe('LINE browser-safe text parser', () => {
|
||||
'03:05pm\tBob\t[Photo]',
|
||||
'03:06pm\t\tAlice joined the group',
|
||||
'',
|
||||
].join('\n')
|
||||
].join('\r\n')
|
||||
|
||||
assert.equal(detectLineText(content, '[LINE] Project Team.txt'), true)
|
||||
assert.equal(detectLineText('2024/01/02 03:04 - Alice: hello', '与Alice的 WhatsApp 聊天.txt'), false)
|
||||
@@ -85,4 +85,25 @@ describe('LINE browser-safe text parser', () => {
|
||||
assert.equal(result.messages.length, 1002)
|
||||
assert.ok(checks > 10)
|
||||
})
|
||||
|
||||
it('classifies prefixed media, links, locations, and system messages', async () => {
|
||||
const content = [
|
||||
'[LINE] Chat history in Project Team',
|
||||
'Saved on: 2024/01/03 09:00',
|
||||
'',
|
||||
'2024.01.02 Tuesday',
|
||||
'03:04\tAlice\t[File] report.pdf',
|
||||
'03:05\tAlice\thttps://example.com',
|
||||
'03:06\tAlice\t[null] https://maps.google.com/example',
|
||||
'03:07\tAlice\tBob left the group',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
const result = await parseLineText(content, '[LINE] Project Team.txt')
|
||||
|
||||
assert.deepEqual(
|
||||
result.messages.map((message) => message.type),
|
||||
[MessageType.FILE, MessageType.LINK, MessageType.LOCATION, MessageType.SYSTEM]
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -75,6 +75,7 @@ const SPECIAL_MESSAGE_TYPES: Record<string, MessageType> = {
|
||||
'[記事本]': MessageType.TEXT,
|
||||
'[ノート]': MessageType.TEXT,
|
||||
}
|
||||
const SPECIAL_MESSAGE_TYPE_ENTRIES = Object.entries(SPECIAL_MESSAGE_TYPES)
|
||||
|
||||
export function detectLineText(head: string, fileName: string): boolean {
|
||||
const basename = getBasename(fileName)
|
||||
@@ -90,20 +91,21 @@ export async function parseLineText(
|
||||
fileName: string,
|
||||
options: LineTextParseOptions = {}
|
||||
): Promise<LineTextParseResult> {
|
||||
const lines = content.split('\n').map((line) => line.replace(/\r$/, ''))
|
||||
const lines = content.split('\n')
|
||||
const memberMap = new Map<string, ParsedMember>()
|
||||
const messages: ParsedMessage[] = []
|
||||
const header = lines.length > 0 ? extractNameFromHeader(lines[0].trim()) : null
|
||||
const header = lines.length > 0 ? extractNameFromHeader(lines[0].replace(/\r$/, '').trim()) : null
|
||||
const chatName = header?.name ?? extractNameFromFileName(fileName)
|
||||
let currentDate: Date | null = null
|
||||
let lastMessage: ParsedMessage | null = null
|
||||
let quotedMultiline = false
|
||||
let lineIndex = header ? 3 : 0
|
||||
let useTabSeparator = Boolean(header)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
if (!header) {
|
||||
for (const line of lines) {
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.replace(/\r$/, '')
|
||||
if (TAB_MESSAGE_PATTERN.test(line)) {
|
||||
useTabSeparator = true
|
||||
break
|
||||
@@ -117,7 +119,7 @@ export async function parseLineText(
|
||||
|
||||
for (; lineIndex < lines.length; lineIndex += 1) {
|
||||
options.checkCancelled?.()
|
||||
const line = lines[lineIndex]
|
||||
const line = lines[lineIndex].replace(/\r$/, '')
|
||||
const date = parseDateLine(line)
|
||||
if (date) {
|
||||
currentDate = date
|
||||
@@ -262,7 +264,7 @@ function parseTime(timeText: string): { hours: number; minutes: number } {
|
||||
}
|
||||
|
||||
function detectMessageType(content: string): MessageType {
|
||||
for (const [pattern, type] of Object.entries(SPECIAL_MESSAGE_TYPES)) {
|
||||
for (const [pattern, type] of SPECIAL_MESSAGE_TYPE_ENTRIES) {
|
||||
if (content === pattern || content.startsWith(pattern)) return type
|
||||
}
|
||||
if (content.startsWith('[null]') && content.includes('maps.google.com')) return MessageType.LOCATION
|
||||
|
||||
@@ -45,7 +45,7 @@ export async function parseQqText(
|
||||
): Promise<QqTextParseResult> {
|
||||
const accumulator = new QqTextAccumulator(fileName)
|
||||
const lines = content.split('\n')
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
options.checkCancelled?.()
|
||||
|
||||
@@ -65,7 +65,7 @@ export async function scanTelegramChatsJson(
|
||||
options.checkCancelled?.()
|
||||
const value = parseTelegramFullExport(content)
|
||||
const chats: TelegramChatInfo[] = []
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 100)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
for (let index = 0; index < value.chats.list.length; index += 1) {
|
||||
options.checkCancelled?.()
|
||||
@@ -124,7 +124,7 @@ async function parseTelegramChat(
|
||||
const memberMap = new Map<string, ParsedMember>()
|
||||
const messages: ParsedMessage[] = []
|
||||
const chatType = mapChatType(value.type)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
for (let index = 0; index < value.messages.length; index += 1) {
|
||||
options.checkCancelled?.()
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
|
||||
|
||||
import { parseWeFlowJson } from './weflow'
|
||||
|
||||
describe('WeFlow browser-safe JSON parser', () => {
|
||||
it('parses messages and only reports intermediate progress at cooperative yield points', async () => {
|
||||
const progress: Array<{ progress: number; messagesProcessed: number }> = []
|
||||
const content = JSON.stringify({
|
||||
weflow: { version: '1.0.0' },
|
||||
session: { wxid: 'group@chatroom', displayName: 'Project Team', type: '群聊' },
|
||||
messages: Array.from({ length: 5 }, (_, index) => ({
|
||||
localId: index + 1,
|
||||
createTime: 1704164645 + index,
|
||||
type: '文本消息',
|
||||
content: `message ${index}`,
|
||||
isSend: index === 0 ? 1 : 0,
|
||||
senderUsername: `member-${index % 2}`,
|
||||
senderDisplayName: `Member ${index % 2}`,
|
||||
})),
|
||||
})
|
||||
|
||||
const result = await parseWeFlowJson(content, 'project.json', {
|
||||
yieldEvery: 2,
|
||||
onProgress: (value) => progress.push(value),
|
||||
})
|
||||
|
||||
assert.deepEqual(result.meta, {
|
||||
name: 'Project Team',
|
||||
platform: KNOWN_PLATFORMS.WECHAT,
|
||||
type: ChatType.GROUP,
|
||||
groupId: 'group@chatroom',
|
||||
groupAvatar: undefined,
|
||||
ownerId: 'member-0',
|
||||
})
|
||||
assert.equal(result.messages.length, 5)
|
||||
assert.equal(result.messages[0]?.type, MessageType.TEXT)
|
||||
assert.deepEqual(
|
||||
progress.map((value) => value.messagesProcessed),
|
||||
[2, 4, 5]
|
||||
)
|
||||
assert.equal(progress.at(-1)?.progress, 1)
|
||||
})
|
||||
})
|
||||
@@ -61,10 +61,16 @@ export async function parseWeFlowJson(
|
||||
const messageValues = Array.isArray(value.messages) ? value.messages : []
|
||||
const members = new Map<string, MemberState>()
|
||||
const messages: ParsedMessage[] = []
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
let ownerId: string | undefined
|
||||
|
||||
for (let index = 0; index < messageValues.length; index += 1) {
|
||||
await cooperate(index, options)
|
||||
options.checkCancelled?.()
|
||||
if (index > 0 && index % yieldEvery === 0) {
|
||||
options.onProgress?.({ progress: index / messageValues.length, messagesProcessed: messages.length })
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
options.checkCancelled?.()
|
||||
}
|
||||
const message = messageValues[index]
|
||||
if (!isRecord(message)) continue
|
||||
|
||||
@@ -92,10 +98,6 @@ export async function parseWeFlowJson(
|
||||
type: convertMessageType(stringValue(message.type)),
|
||||
content: normalizeContent(message.content),
|
||||
})
|
||||
options.onProgress?.({
|
||||
progress: messageValues.length === 0 ? 1 : (index + 1) / messageValues.length,
|
||||
messagesProcessed: messages.length,
|
||||
})
|
||||
}
|
||||
|
||||
options.checkCancelled?.()
|
||||
@@ -183,12 +185,3 @@ function stringValue(value: unknown): string | undefined {
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
async function cooperate(index: number, options: WeFlowJsonParseOptions): Promise<void> {
|
||||
options.checkCancelled?.()
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
if (index > 0 && index % yieldEvery === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
options.checkCancelled?.()
|
||||
}
|
||||
}
|
||||
@@ -108,7 +108,7 @@ export async function parseWhatsAppText(
|
||||
): Promise<WhatsAppTextParseResult> {
|
||||
const accumulator = new WhatsAppTextAccumulator(fileName)
|
||||
const lines = content.split('\n')
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
options.checkCancelled?.()
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, it } from 'node:test'
|
||||
import { MessageType } from '@openchatlab/shared-types'
|
||||
|
||||
import { detectFormat, parseFileSync } from '../index'
|
||||
|
||||
function encodeInstagramText(value: string): string {
|
||||
return Buffer.from(value, 'utf8').toString('latin1')
|
||||
}
|
||||
|
||||
describe('Instagram native parser', () => {
|
||||
it('decodes Instagram text and classifies system messages', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'chatlab-instagram-parser-'))
|
||||
try {
|
||||
const filePath = join(directory, 'message_1.json')
|
||||
writeFileSync(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
participants: [{ name: encodeInstagramText('小明') }, { name: encodeInstagramText('小红') }],
|
||||
messages: [
|
||||
{
|
||||
sender_name: encodeInstagramText('小红'),
|
||||
timestamp_ms: 2000,
|
||||
content: 'Alice added Bob to the group',
|
||||
is_geoblocked_for_viewer: false,
|
||||
},
|
||||
{
|
||||
sender_name: encodeInstagramText('小明'),
|
||||
timestamp_ms: 1000,
|
||||
content: encodeInstagramText('你好'),
|
||||
is_geoblocked_for_viewer: false,
|
||||
},
|
||||
],
|
||||
title: encodeInstagramText('小红'),
|
||||
thread_path: 'inbox/example',
|
||||
}),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
assert.equal(detectFormat(filePath)?.id, 'instagram-native')
|
||||
const result = await parseFileSync(filePath)
|
||||
|
||||
assert.equal(result.meta.name, '小红')
|
||||
assert.deepEqual(
|
||||
result.messages.map((message) => ({
|
||||
sender: message.senderAccountName,
|
||||
type: message.type,
|
||||
content: message.content,
|
||||
})),
|
||||
[
|
||||
{ sender: '小明', type: MessageType.TEXT, content: '你好' },
|
||||
{ sender: '小红', type: MessageType.SYSTEM, content: 'Alice added Bob to the group' },
|
||||
]
|
||||
)
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -106,6 +106,8 @@ interface InstagramData {
|
||||
|
||||
// ==================== 辅助函数 ====================
|
||||
|
||||
const UTF8_DECODER = new TextDecoder('utf-8')
|
||||
|
||||
/**
|
||||
* 解码 Instagram 特殊编码的文本
|
||||
* Instagram 将 UTF-8 字节按 Latin-1 编码后存储
|
||||
@@ -118,7 +120,7 @@ function decodeInstagramText(text: string): string {
|
||||
bytes[i] = text.charCodeAt(i)
|
||||
}
|
||||
// 用 UTF-8 解码
|
||||
return new TextDecoder('utf-8').decode(bytes)
|
||||
return UTF8_DECODER.decode(bytes)
|
||||
} catch {
|
||||
return text // 解码失败则返回原文
|
||||
}
|
||||
@@ -135,23 +137,24 @@ function extractNameFromFilePath(filePath: string): string {
|
||||
/**
|
||||
* 判断是否为系统消息
|
||||
*/
|
||||
const SYSTEM_MESSAGE_PATTERNS = [
|
||||
'You created the group',
|
||||
'created the group',
|
||||
'added',
|
||||
'to the group',
|
||||
'left the group',
|
||||
'removed',
|
||||
'named the group',
|
||||
'changed the group photo',
|
||||
'Reacted',
|
||||
'sent an attachment',
|
||||
'liked a message',
|
||||
'changed the theme',
|
||||
'set the nickname',
|
||||
]
|
||||
|
||||
function isSystemMessage(content: string): boolean {
|
||||
const systemPatterns = [
|
||||
'You created the group',
|
||||
'created the group',
|
||||
'added',
|
||||
'to the group',
|
||||
'left the group',
|
||||
'removed',
|
||||
'named the group',
|
||||
'changed the group photo',
|
||||
'Reacted',
|
||||
'sent an attachment',
|
||||
'liked a message',
|
||||
'changed the theme',
|
||||
'set the nickname',
|
||||
]
|
||||
return systemPatterns.some((p) => content.includes(p))
|
||||
return SYSTEM_MESSAGE_PATTERNS.some((pattern) => content.includes(pattern))
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -286,11 +289,11 @@ async function* parseInstagram(options: ParseOptions): AsyncGenerator<ParseEvent
|
||||
const members = Array.from(memberMap.values())
|
||||
yield { type: 'members', data: members }
|
||||
|
||||
// 处理消息(Instagram 消息是逆序的,需要反转)
|
||||
const reversedMessages = [...data.messages].reverse()
|
||||
// Instagram 消息是逆序的;从尾部遍历即可按时间正序输出,避免复制整份消息数组。
|
||||
const messageBatch: ParsedMessage[] = []
|
||||
|
||||
for (const msg of reversedMessages) {
|
||||
for (let index = data.messages.length - 1; index >= 0; index -= 1) {
|
||||
const msg = data.messages[index]
|
||||
const senderName = decodeInstagramText(msg.sender_name)
|
||||
const timestamp = Math.floor(msg.timestamp_ms / 1000) // 毫秒转秒
|
||||
const type = detectMessageType(msg)
|
||||
@@ -321,7 +324,7 @@ async function* parseInstagram(options: ParseOptions): AsyncGenerator<ParseEvent
|
||||
|
||||
const progress = createProgress(
|
||||
'parsing',
|
||||
Math.floor((messagesProcessed / reversedMessages.length) * totalBytes),
|
||||
Math.floor((messagesProcessed / data.messages.length) * totalBytes),
|
||||
totalBytes,
|
||||
messagesProcessed,
|
||||
`已处理 ${messagesProcessed} 条消息...`
|
||||
|
||||
@@ -23,7 +23,16 @@ describe('ChatLab browser parser', () => {
|
||||
meta: { name: 'Fixture', platform: 'wechat', type: 'group', ownerId: 'alice' },
|
||||
members: [{ platformId: 'alice', accountName: 'Alice' }],
|
||||
messages: [
|
||||
{ sender: 'alice', accountName: 'Alice', timestamp: 1, type: 0, content: 'hello' },
|
||||
{
|
||||
sender: 'alice',
|
||||
accountName: 'Alice',
|
||||
groupNickname: 'A',
|
||||
timestamp: 1,
|
||||
type: 0,
|
||||
content: 'hello',
|
||||
platformMessageId: 'message-1',
|
||||
replyToMessageId: 'message-0',
|
||||
},
|
||||
{ sender: 'bob', accountName: 'Bob', timestamp: 2, type: 0, content: null },
|
||||
],
|
||||
})
|
||||
@@ -44,6 +53,16 @@ describe('ChatLab browser parser', () => {
|
||||
['alice', 'bob']
|
||||
)
|
||||
assert.equal(parsed.messages.length, 2)
|
||||
assert.deepEqual(parsed.messages[0], {
|
||||
senderPlatformId: 'alice',
|
||||
senderAccountName: 'Alice',
|
||||
senderGroupNickname: 'A',
|
||||
timestamp: 1,
|
||||
type: 0,
|
||||
content: 'hello',
|
||||
platformMessageId: 'message-1',
|
||||
replyToMessageId: 'message-0',
|
||||
})
|
||||
assert.equal(parsed.messages[1].senderPlatformId, 'bob')
|
||||
})
|
||||
|
||||
@@ -114,4 +133,45 @@ describe('ChatLab browser parser', () => {
|
||||
/cancelled/
|
||||
)
|
||||
})
|
||||
|
||||
it('throttles JSON and JSONL progress updates to cooperative yield points', async () => {
|
||||
const messages = Array.from({ length: 5 }, (_, index) => ({
|
||||
sender: `member-${index}`,
|
||||
accountName: `Member ${index}`,
|
||||
timestamp: index + 1,
|
||||
type: 0,
|
||||
content: 'message',
|
||||
}))
|
||||
const jsonProgress: number[] = []
|
||||
await parseChatLabSource(
|
||||
source(
|
||||
'progress.json',
|
||||
JSON.stringify({
|
||||
chatlab: { version: '1', exportedAt: 1 },
|
||||
meta: { name: 'Progress', platform: 'unknown', type: 'group' },
|
||||
messages,
|
||||
})
|
||||
),
|
||||
{ yieldEvery: 2, onProgress: (value) => jsonProgress.push(value.messagesProcessed) }
|
||||
)
|
||||
|
||||
const jsonlProgress: number[] = []
|
||||
await parseChatLabSource(
|
||||
source(
|
||||
'progress.jsonl',
|
||||
[
|
||||
JSON.stringify({
|
||||
_type: 'header',
|
||||
chatlab: { version: '1', exportedAt: 1 },
|
||||
meta: { name: 'Progress', platform: 'unknown', type: 'group' },
|
||||
}),
|
||||
...messages.map((message) => JSON.stringify({ _type: 'message', ...message })),
|
||||
].join('\n')
|
||||
),
|
||||
{ yieldEvery: 2, onProgress: (value) => jsonlProgress.push(value.messagesProcessed) }
|
||||
)
|
||||
|
||||
assert.deepEqual(jsonProgress, [2, 4, 5])
|
||||
assert.deepEqual(jsonlProgress, [0, 2, 4, 5])
|
||||
})
|
||||
})
|
||||
@@ -116,17 +116,23 @@ async function parseJson(content: string, options: ParseChatLabSourceOptions): P
|
||||
}
|
||||
|
||||
const messages: BrowserParsedMessage[] = []
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
for (let index = 0; index < root.messages.length; index += 1) {
|
||||
await cooperate(index, options)
|
||||
options.checkCancelled?.()
|
||||
if (index > 0 && index % yieldEvery === 0) {
|
||||
options.onProgress?.({
|
||||
stage: 'parsing',
|
||||
progress: index / root.messages.length,
|
||||
messagesProcessed: messages.length,
|
||||
})
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
options.checkCancelled?.()
|
||||
}
|
||||
messages.push(parseMessage(root.messages[index], `messages[${index}]`))
|
||||
options.onProgress?.({
|
||||
stage: 'parsing',
|
||||
progress: root.messages.length === 0 ? 1 : (index + 1) / root.messages.length,
|
||||
messagesProcessed: index + 1,
|
||||
})
|
||||
}
|
||||
|
||||
options.checkCancelled?.()
|
||||
options.onProgress?.({ stage: 'parsing', progress: 1, messagesProcessed: messages.length })
|
||||
return {
|
||||
formatId: 'chatlab',
|
||||
meta,
|
||||
@@ -142,6 +148,7 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
|
||||
let firstRecordSeen = false
|
||||
let cursor = 0
|
||||
let lineNumber = 0
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 5000)
|
||||
|
||||
while (cursor <= content.length) {
|
||||
const nextBreak = content.indexOf('\n', cursor)
|
||||
@@ -149,9 +156,18 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
|
||||
const line = content.slice(cursor, end).replace(/\r$/, '').trim()
|
||||
lineNumber += 1
|
||||
cursor = nextBreak === -1 ? content.length + 1 : nextBreak + 1
|
||||
options.checkCancelled?.()
|
||||
if (lineNumber > 0 && lineNumber % yieldEvery === 0) {
|
||||
options.onProgress?.({
|
||||
stage: 'parsing',
|
||||
progress: content.length === 0 ? 1 : Math.min(1, cursor / content.length),
|
||||
messagesProcessed: messages.length,
|
||||
})
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
options.checkCancelled?.()
|
||||
}
|
||||
if (!line || line.startsWith('#')) continue
|
||||
|
||||
await cooperate(lineNumber, options)
|
||||
let value: unknown
|
||||
try {
|
||||
value = JSON.parse(line)
|
||||
@@ -182,11 +198,6 @@ async function parseJsonl(content: string, options: ParseChatLabSourceOptions):
|
||||
break
|
||||
case 'message':
|
||||
messages.push(parseMessage(value, `line ${lineNumber}`))
|
||||
options.onProgress?.({
|
||||
stage: 'parsing',
|
||||
progress: content.length === 0 ? 1 : Math.min(1, cursor / content.length),
|
||||
messagesProcessed: messages.length,
|
||||
})
|
||||
break
|
||||
default:
|
||||
throw new WebRuntimeError('INVALID_IMPORT_FILE', `Unknown ChatLab JSONL record type at line ${lineNumber}`)
|
||||
@@ -244,16 +255,20 @@ function parseMessage(value: unknown, path: string): BrowserParsedMessage {
|
||||
const content = value.content
|
||||
if (content !== null && typeof content !== 'string') throw invalidField(`${path}.content`, 'must be a string or null')
|
||||
|
||||
return omitUndefined({
|
||||
const message: BrowserParsedMessage = {
|
||||
senderPlatformId: sender,
|
||||
senderAccountName: optionalString(value.accountName, `${path}.accountName`) ?? sender,
|
||||
senderGroupNickname: optionalString(value.groupNickname, `${path}.groupNickname`),
|
||||
timestamp,
|
||||
type,
|
||||
content,
|
||||
platformMessageId: optionalString(value.platformMessageId, `${path}.platformMessageId`),
|
||||
replyToMessageId: optionalString(value.replyToMessageId, `${path}.replyToMessageId`),
|
||||
})
|
||||
}
|
||||
const senderGroupNickname = optionalString(value.groupNickname, `${path}.groupNickname`)
|
||||
const platformMessageId = optionalString(value.platformMessageId, `${path}.platformMessageId`)
|
||||
const replyToMessageId = optionalString(value.replyToMessageId, `${path}.replyToMessageId`)
|
||||
if (senderGroupNickname !== undefined) message.senderGroupNickname = senderGroupNickname
|
||||
if (platformMessageId !== undefined) message.platformMessageId = platformMessageId
|
||||
if (replyToMessageId !== undefined) message.replyToMessageId = replyToMessageId
|
||||
return message
|
||||
}
|
||||
|
||||
function mergeInferredMembers(
|
||||
@@ -272,15 +287,6 @@ function mergeInferredMembers(
|
||||
return [...byId.values()]
|
||||
}
|
||||
|
||||
async function cooperate(index: number, options: ParseChatLabSourceOptions): Promise<void> {
|
||||
options.checkCancelled?.()
|
||||
const yieldEvery = Math.max(1, options.yieldEvery ?? 1000)
|
||||
if (index > 0 && index % yieldEvery === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
options.checkCancelled?.()
|
||||
}
|
||||
}
|
||||
|
||||
function firstMeaningfulLine(content: string): string | undefined {
|
||||
return content
|
||||
.split(/\r?\n/)
|
||||
|
||||
@@ -34,9 +34,12 @@ describe('browser Rust WASM parser', () => {
|
||||
{
|
||||
sender: 'alice',
|
||||
accountName: 'Alice',
|
||||
groupNickname: 'A',
|
||||
timestamp: 1,
|
||||
type: 0,
|
||||
content: 'hello chatlab',
|
||||
platformMessageId: 'message-1',
|
||||
replyToMessageId: 'message-0',
|
||||
},
|
||||
],
|
||||
})
|
||||
@@ -54,6 +57,16 @@ describe('browser Rust WASM parser', () => {
|
||||
chatlab.messages.map((message) => message.content),
|
||||
['hello chatlab']
|
||||
)
|
||||
assert.deepEqual(chatlab.messages[0], {
|
||||
senderPlatformId: 'alice',
|
||||
senderAccountName: 'Alice',
|
||||
senderGroupNickname: 'A',
|
||||
timestamp: 1,
|
||||
type: 0,
|
||||
content: 'hello chatlab',
|
||||
platformMessageId: 'message-1',
|
||||
replyToMessageId: 'message-0',
|
||||
})
|
||||
assert.deepEqual(chatlab, chatlabTs)
|
||||
|
||||
const weflowSource = source('weflow.json', {
|
||||
|
||||
@@ -99,7 +99,7 @@ export async function parseWithWasm(
|
||||
const batchJson = parser.take_batch_json(batchSize)
|
||||
if (batchJson === undefined) break
|
||||
const batch = JSON.parse(batchJson) as WasmNativeMessage[]
|
||||
messages.push(...batch.map((message) => mapMessage(message, formatId)))
|
||||
for (const message of batch) messages.push(mapMessage(message, formatId))
|
||||
options.onProgress?.({
|
||||
stage: 'parsing',
|
||||
progress: summary.messageCount === 0 ? 1 : messages.length / summary.messageCount,
|
||||
@@ -218,21 +218,32 @@ function mapMembers(
|
||||
}
|
||||
|
||||
function mapMessage(message: WasmNativeMessage, formatId: BrowserWasmFormatId): BrowserParsedMessage {
|
||||
const mapped = {
|
||||
platformMessageId: optionalString(message.platformMessageId),
|
||||
const platformMessageId = optionalString(message.platformMessageId)
|
||||
const senderGroupNickname = optionalString(message.senderGroupNickname)
|
||||
const replyToMessageId = optionalString(message.replyToMessageId)
|
||||
if (formatId === 'weflow') {
|
||||
return {
|
||||
platformMessageId,
|
||||
senderPlatformId: message.senderPlatformId,
|
||||
senderAccountName: message.senderAccountName,
|
||||
senderGroupNickname,
|
||||
timestamp: message.timestamp as number,
|
||||
type: message.messageType,
|
||||
content: message.content ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
const mapped: BrowserParsedMessage = {
|
||||
senderPlatformId: message.senderPlatformId,
|
||||
senderAccountName: message.senderAccountName,
|
||||
senderGroupNickname: optionalString(message.senderGroupNickname),
|
||||
timestamp: message.timestamp as number,
|
||||
type: message.messageType,
|
||||
content: message.content ?? null,
|
||||
replyToMessageId: optionalString(message.replyToMessageId),
|
||||
}
|
||||
if (formatId === 'weflow') {
|
||||
const { replyToMessageId: _replyToMessageId, ...weflowMessage } = mapped
|
||||
return weflowMessage
|
||||
}
|
||||
return compact(mapped)
|
||||
if (platformMessageId !== undefined) mapped.platformMessageId = platformMessageId
|
||||
if (senderGroupNickname !== undefined) mapped.senderGroupNickname = senderGroupNickname
|
||||
if (replyToMessageId !== undefined) mapped.replyToMessageId = replyToMessageId
|
||||
return mapped
|
||||
}
|
||||
|
||||
function optionalString(value: unknown): string | undefined {
|
||||
|
||||
Reference in new issue
Block a user