From 1e7aefe08b74157134f4ee3191c41c61a76ffbff Mon Sep 17 00:00:00 2001 From: digua Date: Thu, 2 Jul 2026 23:29:11 +0800 Subject: [PATCH] feat(import): add Rust native parsing kernels for WeFlow and ChatLab JSON Add @openchatlab/parser-native (napi-rs v3) with a shared byte-level JSON scanner and two format kernels. Parsing large exports is 4-10x faster in real-data benchmarks. The native module is an optional local build; the pure-TS parsers remain as automatic fallback when it is unavailable, disabled via CHATLAB_DISABLE_NATIVE_PERF=1, or rejects off-spec files. The ChatLab kernel also parses the members array structurally, fixing the TS head-regex heuristic that silently dropped avatars/roles for files with nested arrays or long member lists. Co-authored-by: Cursor --- .gitignore | 3 + AGENTS.md | 1 + apps/cli/tsup.config.ts | 6 +- apps/desktop/electron-builder.yml | 1 + apps/desktop/package.json | 1 + package.json | 1 + packages/parser-native/.gitignore | 2 + packages/parser-native/Cargo.lock | 358 ++++++ packages/parser-native/Cargo.toml | 23 + packages/parser-native/build.rs | 3 + packages/parser-native/index.d.ts | 114 ++ packages/parser-native/index.js | 591 +++++++++ packages/parser-native/package.json | 19 + packages/parser-native/src/chatlab.rs | 504 ++++++++ packages/parser-native/src/jsutil.rs | 32 + packages/parser-native/src/lib.rs | 442 +++++++ packages/parser-native/src/scanner.rs | 331 +++++ packages/parser-native/src/weflow.rs | 432 +++++++ packages/parser/package.json | 1 + packages/parser/src/formats/chatlab.ts | 8 +- packages/parser/src/formats/weflow.ts | 6 +- .../parser/src/formats/ycccccccy-echotrace.ts | 4 +- .../parser/src/native/chatlab-native.test.ts | 220 ++++ packages/parser/src/native/chatlab-native.ts | 153 +++ packages/parser/src/native/loader.ts | 44 + .../parser/src/native/weflow-native.test.ts | 264 ++++ packages/parser/src/native/weflow-native.ts | 138 +++ pnpm-lock.yaml | 1102 +++++++++++++++++ scripts/bench-weflow-parser.mts | 155 +++ 29 files changed, 4953 insertions(+), 6 deletions(-) create mode 100644 packages/parser-native/.gitignore create mode 100644 packages/parser-native/Cargo.lock create mode 100644 packages/parser-native/Cargo.toml create mode 100644 packages/parser-native/build.rs create mode 100644 packages/parser-native/index.d.ts create mode 100644 packages/parser-native/index.js create mode 100644 packages/parser-native/package.json create mode 100644 packages/parser-native/src/chatlab.rs create mode 100644 packages/parser-native/src/jsutil.rs create mode 100644 packages/parser-native/src/lib.rs create mode 100644 packages/parser-native/src/scanner.rs create mode 100644 packages/parser-native/src/weflow.rs create mode 100644 packages/parser/src/native/chatlab-native.test.ts create mode 100644 packages/parser/src/native/chatlab-native.ts create mode 100644 packages/parser/src/native/loader.ts create mode 100644 packages/parser/src/native/weflow-native.test.ts create mode 100644 packages/parser/src/native/weflow-native.ts create mode 100644 scripts/bench-weflow-parser.mts diff --git a/.gitignore b/.gitignore index 76a0865a..259c9fdc 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ yarn.lock .docs .playwright-mcp/ + +# Local real-chat samples for manual testing (privacy: never commit) +weixin-demo.json diff --git a/AGENTS.md b/AGENTS.md index 54f4fda9..bfc97497 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ - `packages/node-runtime/`：Node.js 运行时能力，包括 SQLite 适配、数据库迁移、AI 管理、导出、缓存和数据目录 - `packages/tools/`：AI 工具定义、工具 registry 和数据访问 provider - `packages/parser/`：聊天导出格式解析器和格式识别 +- `packages/parser-native/`：napi-rs Rust 原生解析内核（可选本地构建，未构建时 parser 自动回退 TS 实现） - `packages/http-routes/`：Electron 和 CLI Web 复用的 HTTP route - `docs/`：公开文档站源码；`.docs/`：私有开发上下文和任务记录，不作为公开 PR 理解前提 - 更细的架构说明继续以 `docs/cn/contributing/development.md` 和 `.docs/README.md` 为准，不在根 `AGENTS.md` 里重复维护 diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 09a70a1c..501de58b 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -19,8 +19,10 @@ export default defineConfig({ define: { 'process.env.APTABASE_APP_KEY': JSON.stringify(process.env.APTABASE_APP_KEY || ''), }, - noExternal: [/^@openchatlab\//, 'chatlab-mcp', 'stream-json'], - external: ['better-sqlite3', '@node-rs/jieba'], + noExternal: [/^@openchatlab\/(?!parser-native)/, 'chatlab-mcp', 'stream-json'], + // parser-native 是本地构建的可选 Rust 内核：不打包也不声明依赖， + // 运行时 require 失败会自动回退 TS 解析器。 + external: ['better-sqlite3', '@node-rs/jieba', '@openchatlab/parser-native'], banner: { js: [ "import { createRequire as __createRequire } from 'module';", diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 76fe5d26..fef66536 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -40,6 +40,7 @@ asarUnpack: - node_modules/sqlite-vec*/** - node_modules/onnxruntime-node/** - node_modules/@img/** + - node_modules/@openchatlab/parser-native/** # Windows 平台配置 win: diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 96382d59..a385e99e 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -19,6 +19,7 @@ "@fastify/multipart": "^10.0.0", "@huggingface/transformers": "^4.2.0", "@node-rs/jieba": "^2.0.1", + "@openchatlab/parser-native": "workspace:*", "better-sqlite3": "^12.4.6", "electron-updater": "^6.6.2", "i18next": "^25.8.5", diff --git a/package.json b/package.json index cc87ed7a..8d7d54ce 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "scripts": { "dev": "node scripts/dev-select.mjs", "dev:desktop": "pnpm --filter @openchatlab/desktop dev", + "build:native": "pnpm --filter @openchatlab/parser-native build", "build:desktop": "pnpm --filter @openchatlab/desktop build", "build:mac": "pnpm --filter @openchatlab/desktop build:mac", "build:win": "pnpm --filter @openchatlab/desktop build:win", diff --git a/packages/parser-native/.gitignore b/packages/parser-native/.gitignore new file mode 100644 index 00000000..89fb9af0 --- /dev/null +++ b/packages/parser-native/.gitignore @@ -0,0 +1,2 @@ +target/ +*.node diff --git a/packages/parser-native/Cargo.lock b/packages/parser-native/Cargo.lock new file mode 100644 index 00000000..589971ad --- /dev/null +++ b/packages/parser-native/Cargo.lock @@ -0,0 +1,358 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chatlab-parser-native" +version = "0.1.0" +dependencies = [ + "memchr", + "napi", + "napi-build", + "napi-derive", + "serde", + "serde_json", +] + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "ctor" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01334b89b69ff726750c5ce5073fc8bd860e99aa9a8fc5ca11b04730e3aee97a" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "napi" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd366ba6a12e4bdbc360e1ad4e1f01da40c4395eef1f9e872d88f8b135e8048" +dependencies = [ + "bitflags", + "ctor", + "futures", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", +] + +[[package]] +name = "napi-build" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1" + +[[package]] +name = "napi-derive" +version = "3.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6730ee4e7b335eac6d7cf10fc2525d92743bd4713a13cb2e6667f0e97322d7c3" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5ccc18b1b16d1049dcbd3e1a21fdfbf3ce720b8944979c5e1a76d4e30d9f9f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f5bcdf71abd3a50d00b49c1c2c75251cb3c913777d6139cd37dabc093a5e400" +dependencies = [ + "libloading", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/packages/parser-native/Cargo.toml b/packages/parser-native/Cargo.toml new file mode 100644 index 00000000..77da84ff --- /dev/null +++ b/packages/parser-native/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "chatlab-parser-native" +version = "0.1.0" +edition = "2021" +license = "AGPL-3.0" +description = "ChatLab native chat-export parsing kernels (napi-rs)" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +napi = { version = "3", features = ["napi8"] } +napi-derive = "3" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order", "raw_value"] } +memchr = "2" + +[build-dependencies] +napi-build = "2" + +[profile.release] +lto = true +strip = "symbols" diff --git a/packages/parser-native/build.rs b/packages/parser-native/build.rs new file mode 100644 index 00000000..0f1b0100 --- /dev/null +++ b/packages/parser-native/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/packages/parser-native/index.d.ts b/packages/parser-native/index.d.ts new file mode 100644 index 00000000..f5d70ff4 --- /dev/null +++ b/packages/parser-native/index.d.ts @@ -0,0 +1,114 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ +/** + * Rust ChatLab JSON parser. Strict about the format spec: off-spec files + * error out and the JS side falls back to the pure-TS parser. + * Usage from JS: `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. + */ +export declare class ChatlabParser { + constructor(filePath: string) + /** Run the full parse on the libuv thread pool. */ + parse(): Promise + /** Poll progress while `parse()` is pending. */ + progress(): NativeParseProgress + getMeta(): NativeChatlabMeta + /** Move members out to JS. Callable once. */ + takeMembers(): Array + /** Move the next batch of messages out to JS; null when exhausted. */ + takeBatch(size: number): Array | null +} + +/** + * Rust WeFlow parser. Usage from JS: + * `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. + */ +export declare class WeflowParser { + constructor(filePath: string) + /** Run the full parse on the libuv thread pool. */ + parse(): Promise + /** Poll progress while `parse()` is pending. */ + progress(): NativeParseProgress + getMeta(): NativeWeflowMeta + /** Move members (with avatars) out to JS. Callable once. */ + takeMembers(): Array + /** Move the next batch of messages out to JS; null when exhausted. */ + takeBatch(size: number): Array | null +} + +export interface NativeChatlabMember { + platformId: string + accountName: string + groupNickname?: string + avatar?: string + roles?: Array +} + +export interface NativeChatlabMessage { + platformMessageId?: string + senderPlatformId: string + senderAccountName: string + senderGroupNickname?: string + timestamp: number + /** Numeric MessageType enum value from shared-types. */ + messageType: number + /** undefined here means the source `content` was JSON null. */ + content?: string + replyToMessageId?: string +} + +export interface NativeChatlabMeta { + name: string + /** Passthrough of `meta.type` ("group" when missing), matching the TS parser. */ + chatType: string + /** Passthrough of `meta.platform` ("unknown" when missing). */ + platform: string + groupId?: string + groupAvatar?: string + /** + * true when members came from the top-level `members` array; + * false when collected from messages (TS emits different object shapes). + */ + membersFromHead: boolean + messageCount: number + memberCount: number +} + +export interface NativeMemberRole { + id: string + /** Present only when the source role object had a `name` key. */ + name?: string +} + +export interface NativeParsedMember { + platformId: string + accountName: string + avatar?: string +} + +export interface NativeParsedMessage { + platformMessageId: string + senderPlatformId: string + senderAccountName: string + /** null when the source `createTime` was JSON null (importer skips those). */ + timestamp?: number + /** Numeric MessageType enum value from shared-types. */ + messageType: number + content?: string +} + +export interface NativeParseProgress { + bytesRead: number + totalBytes: number + messagesProcessed: number +} + +export interface NativeWeflowMeta { + name: string + /** "group" | "private" (ChatType enum string values) */ + chatType: string + groupId?: string + groupAvatar?: string + ownerId?: string + messageCount: number + memberCount: number +} diff --git a/packages/parser-native/index.js b/packages/parser-native/index.js new file mode 100644 index 00000000..08d8921c --- /dev/null +++ b/packages/parser-native/index.js @@ -0,0 +1,591 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* auto-generated by NAPI-RS */ + +const { readFileSync } = require('fs') +let nativeBinding = null +const loadErrors = [] + +const isMusl = () => { + let musl = false + if (process.platform === 'linux') { + musl = isMuslFromFilesystem() + if (musl === null) { + musl = isMuslFromReport() + } + if (musl === null) { + musl = isMuslFromChildProcess() + } + } + return musl +} + +const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-') + +const isMuslFromFilesystem = () => { + try { + return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl') + } catch { + return null + } +} + +const isMuslFromReport = () => { + let report = null + if (process.report && typeof process.report.getReport === 'function') { + process.report.excludeNetwork = true + report = process.report.getReport() + } + if (!report) { + return null + } + if (report.header && report.header.glibcVersionRuntime) { + return false + } + if (Array.isArray(report.sharedObjects)) { + if (report.sharedObjects.some(isFileMusl)) { + return true + } + } + return false +} + +const isMuslFromChildProcess = () => { + try { + return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl') + } catch (e) { + // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false + return false + } +} + +function requireNative() { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + try { + return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH); + } catch (err) { + loadErrors.push(err) + } + } else if (process.platform === 'android') { + if (process.arch === 'arm64') { + try { + return require('./chatlab-parser-native.android-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-android-arm64') + const bindingPackageVersion = require('@openchatlab/parser-native-android-arm64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./chatlab-parser-native.android-arm-eabi.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-android-arm-eabi') + const bindingPackageVersion = require('@openchatlab/parser-native-android-arm-eabi/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`)) + } + } else if (process.platform === 'win32') { + if (process.arch === 'x64') { + if ((process.config && process.config.variables && process.config.variables.shlib_suffix === 'dll.a') || (process.config && process.config.variables && process.config.variables.node_target_type === 'shared_library')) { + try { + return require('./chatlab-parser-native.win32-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-win32-x64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-win32-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.win32-x64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-win32-x64-msvc') + const bindingPackageVersion = require('@openchatlab/parser-native-win32-x64-msvc/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ia32') { + try { + return require('./chatlab-parser-native.win32-ia32-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-win32-ia32-msvc') + const bindingPackageVersion = require('@openchatlab/parser-native-win32-ia32-msvc/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./chatlab-parser-native.win32-arm64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-win32-arm64-msvc') + const bindingPackageVersion = require('@openchatlab/parser-native-win32-arm64-msvc/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`)) + } + } else if (process.platform === 'darwin') { + try { + return require('./chatlab-parser-native.darwin-universal.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-darwin-universal') + const bindingPackageVersion = require('@openchatlab/parser-native-darwin-universal/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + if (process.arch === 'x64') { + try { + return require('./chatlab-parser-native.darwin-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-darwin-x64') + const bindingPackageVersion = require('@openchatlab/parser-native-darwin-x64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./chatlab-parser-native.darwin-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-darwin-arm64') + const bindingPackageVersion = require('@openchatlab/parser-native-darwin-arm64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`)) + } + } else if (process.platform === 'freebsd') { + if (process.arch === 'x64') { + try { + return require('./chatlab-parser-native.freebsd-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-freebsd-x64') + const bindingPackageVersion = require('@openchatlab/parser-native-freebsd-x64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./chatlab-parser-native.freebsd-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-freebsd-arm64') + const bindingPackageVersion = require('@openchatlab/parser-native-freebsd-arm64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`)) + } + } else if (process.platform === 'linux') { + if (process.arch === 'x64') { + if (isMusl()) { + try { + return require('./chatlab-parser-native.linux-x64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-x64-musl') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-x64-musl/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.linux-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-x64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm64') { + if (isMusl()) { + try { + return require('./chatlab-parser-native.linux-arm64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-arm64-musl') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-arm64-musl/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.linux-arm64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-arm64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-arm64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm') { + if (isMusl()) { + try { + return require('./chatlab-parser-native.linux-arm-musleabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-arm-musleabihf') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-arm-musleabihf/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.linux-arm-gnueabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-arm-gnueabihf') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-arm-gnueabihf/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'loong64') { + if (isMusl()) { + try { + return require('./chatlab-parser-native.linux-loong64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-loong64-musl') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-loong64-musl/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.linux-loong64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-loong64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-loong64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'riscv64') { + if (isMusl()) { + try { + return require('./chatlab-parser-native.linux-riscv64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-riscv64-musl') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-riscv64-musl/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./chatlab-parser-native.linux-riscv64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-riscv64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-riscv64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ppc64') { + try { + return require('./chatlab-parser-native.linux-ppc64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-ppc64-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-ppc64-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 's390x') { + try { + return require('./chatlab-parser-native.linux-s390x-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-linux-s390x-gnu') + const bindingPackageVersion = require('@openchatlab/parser-native-linux-s390x-gnu/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`)) + } + } else if (process.platform === 'openharmony') { + if (process.arch === 'arm64') { + try { + return require('./chatlab-parser-native.openharmony-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-openharmony-arm64') + const bindingPackageVersion = require('@openchatlab/parser-native-openharmony-arm64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'x64') { + try { + return require('./chatlab-parser-native.openharmony-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-openharmony-x64') + const bindingPackageVersion = require('@openchatlab/parser-native-openharmony-x64/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./chatlab-parser-native.openharmony-arm.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@openchatlab/parser-native-openharmony-arm') + const bindingPackageVersion = require('@openchatlab/parser-native-openharmony-arm/package.json').version + if (bindingPackageVersion !== '0.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`)) + } + } else { + loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`)) + } +} + +nativeBinding = requireNative() + +// NAPI_RS_FORCE_WASI is a tri-state flag: +// unset / any other value → native binding preferred, WASI is only a fallback +// 'true' → force WASI fallback even if native loaded +// 'error' → force WASI and throw if no WASI binding is found +// Treating any non-empty string as truthy (the historical behavior) meant +// NAPI_RS_FORCE_WASI=false, NAPI_RS_FORCE_WASI=0, etc. inadvertently triggered +// the WASI path, causing ENOENT for packages shipped without a .wasi.cjs file. +const forceWasi = + process.env.NAPI_RS_FORCE_WASI === 'true' || process.env.NAPI_RS_FORCE_WASI === 'error' + +if (!nativeBinding || forceWasi) { + let wasiBinding = null + let wasiBindingError = null + try { + wasiBinding = require('./chatlab-parser-native.wasi.cjs') + nativeBinding = wasiBinding + } catch (err) { + if (forceWasi) { + wasiBindingError = err + } + } + if (!nativeBinding || forceWasi) { + try { + wasiBinding = require('@openchatlab/parser-native-wasm32-wasi') + nativeBinding = wasiBinding + } catch (err) { + if (forceWasi) { + if (!wasiBindingError) { + wasiBindingError = err + } else { + wasiBindingError.cause = err + } + loadErrors.push(err) + } + } + } + if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) { + const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error') + error.cause = wasiBindingError + throw error + } +} + +if (!nativeBinding) { + if (loadErrors.length > 0) { + const error = new Error( + `Cannot find native binding. ` + + `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` + + 'Please try `npm i` again after removing both package-lock.json and node_modules directory.', + ) + // assign instead of the `new Error(message, { cause })` options form, + // which Node < 16.9 silently ignores + error.cause = loadErrors.reduce((err, cur) => { + cur.cause = err + return cur + }) + throw error + } + throw new Error(`Failed to load native binding`) +} + +module.exports = nativeBinding +module.exports.ChatlabParser = nativeBinding.ChatlabParser +module.exports.WeflowParser = nativeBinding.WeflowParser diff --git a/packages/parser-native/package.json b/packages/parser-native/package.json new file mode 100644 index 00000000..eddabcdd --- /dev/null +++ b/packages/parser-native/package.json @@ -0,0 +1,19 @@ +{ + "name": "@openchatlab/parser-native", + "version": "0.0.0", + "private": true, + "description": "ChatLab Rust 原生解析内核（napi-rs，一期覆盖 WeFlow 格式）", + "main": "index.js", + "types": "index.d.ts", + "napi": { + "binaryName": "chatlab-parser-native" + }, + "scripts": { + "build": "napi build --platform --release", + "build:debug": "napi build --platform", + "test": "cargo test" + }, + "devDependencies": { + "@napi-rs/cli": "^3.0.0" + } +} diff --git a/packages/parser-native/src/chatlab.rs b/packages/parser-native/src/chatlab.rs new file mode 100644 index 00000000..5b749f1d --- /dev/null +++ b/packages/parser-native/src/chatlab.rs @@ -0,0 +1,504 @@ +//! ChatLab JSON format kernel. +//! +//! Mirrors `packages/parser/src/formats/chatlab.ts` for spec-compliant files. +//! Contract: this kernel is strict — any field that deviates from the ChatLab +//! format spec (wrong type, unexpected null) returns an error, and the TS +//! wrapper falls back to the pure-TS parser which replicates all passthrough +//! quirks. Known intentional improvements over the TS reference (only reachable +//! on degenerate inputs): meta/members are parsed structurally instead of via +//! the TS 200KB-head regex heuristic, so files whose meta exceeds the head +//! window or whose members contain nested arrays keep their real data here. + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::jsutil::{extract_name_from_file_path, non_empty_str}; +use crate::scanner::{for_each_array_element, walk_top_level, ScanError, ScanResult}; + +pub struct RoleOut { + pub id: String, + pub name: Option, +} + +pub struct ChatlabMemberOut { + pub platform_id: String, + pub account_name: String, + pub group_nickname: Option, + pub avatar: Option, + pub roles: Option>, +} + +#[derive(Default)] +pub struct ChatlabMessageOut { + pub platform_message_id: Option, + pub sender_platform_id: String, + pub sender_account_name: String, + pub sender_group_nickname: Option, + pub timestamp: f64, + pub message_type: u32, + /// None means the source `content` was JSON null. + pub content: Option, + pub reply_to_message_id: Option, +} + +pub struct ChatlabOutput { + pub name: String, + /// Passthrough of `meta.type` ("group" when missing/empty), like the TS parser. + pub chat_type: String, + /// Passthrough of `meta.platform` ("unknown" when missing/empty). + pub platform: String, + pub group_id: Option, + pub group_avatar: Option, + /// true: members came from the top-level `members` array (full shape); + /// false: collected from messages (id/name/nickname only), as in TS. + pub members_from_head: bool, + pub members: Vec, + pub messages: Vec, +} + +fn strict(msg: impl Into) -> ScanError { + ScanError { + message: msg.into(), + offset: 0, + } +} + +/// Required string field (any content, including empty). +fn require_str(obj: &serde_json::Map, key: &str, ctx: &str) -> ScanResult { + match obj.get(key) { + Some(Value::String(s)) => Ok(s.clone()), + other => Err(strict(format!( + "{ctx}: field '{key}' must be a string, got {}", + type_name(other) + ))), + } +} + +/// Optional string field: missing -> None; string -> Some; anything else (incl. null) -> strict error. +fn optional_str( + obj: &serde_json::Map, + key: &str, + ctx: &str, +) -> ScanResult> { + match obj.get(key) { + None => Ok(None), + Some(Value::String(s)) => Ok(Some(s.clone())), + other => Err(strict(format!( + "{ctx}: field '{key}' must be a string when present, got {}", + type_name(other) + ))), + } +} + +fn type_name(value: Option<&Value>) -> &'static str { + match value { + None => "missing", + Some(Value::Null) => "null", + Some(Value::Bool(_)) => "boolean", + Some(Value::Number(_)) => "number", + Some(Value::String(_)) => "string", + Some(Value::Array(_)) => "array", + Some(Value::Object(_)) => "object", + } +} + +struct MetaOut { + name: String, + platform: String, + chat_type: String, + group_id: Option, + group_avatar: Option, +} + +fn parse_meta(meta_raw: Option<&[u8]>, file_path: &str) -> ScanResult { + // A missing or non-object meta puts the TS parser on its defaults-only + // path, which emits a meta object without groupId/groupAvatar keys. + // Delegate that shape to the TS fallback instead of replicating it. + let raw = meta_raw.ok_or_else(|| strict("meta object missing"))?; + let meta: serde_json::Map = match serde_json::from_slice::(raw) { + Ok(Value::Object(map)) => map, + Ok(_) => return Err(strict("meta must be an object")), + Err(err) => return Err(strict(format!("invalid meta object: {err}"))), + }; + + let mut name = "未知群聊".to_string(); + let mut platform = "unknown".to_string(); + let mut chat_type = "group".to_string(); + + // `metaObj.name || '未知群聊'` — non-empty string wins; a non-string + // truthy name is off-spec, so be strict about it. + match meta.get("name") { + None | Some(Value::Null) => {} + Some(Value::String(s)) => { + if !s.is_empty() { + name = s.clone(); + } + } + other => { + return Err(strict(format!( + "meta.name must be a string, got {}", + type_name(other) + ))) + } + } + if let Some(p) = non_empty_str(meta.get("platform")) { + platform = p.to_string(); + } else if !matches!( + meta.get("platform"), + None | Some(Value::Null) | Some(Value::String(_)) + ) { + return Err(strict("meta.platform must be a string")); + } + if let Some(t) = non_empty_str(meta.get("type")) { + chat_type = t.to_string(); + } else if !matches!( + meta.get("type"), + None | Some(Value::Null) | Some(Value::String(_)) + ) { + return Err(strict("meta.type must be a string")); + } + let group_id = optional_str(&meta, "groupId", "meta")?; + let group_avatar = optional_str(&meta, "groupAvatar", "meta")?; + + // Filename fallback fires whenever the name resolved to the default, + // mirroring `if (meta.name === '未知群聊')` in the TS parser. + if name == "未知群聊" { + name = extract_name_from_file_path(file_path, "未知群聊"); + } + + Ok(MetaOut { + name, + platform, + chat_type, + group_id, + group_avatar, + }) +} + +fn parse_member(element: &[u8]) -> ScanResult { + let value: Value = + serde_json::from_slice(element).map_err(|err| strict(format!("invalid member: {err}")))?; + let obj = value + .as_object() + .ok_or_else(|| strict("member must be an object"))?; + + let roles = match obj.get("roles") { + None => None, + Some(Value::Array(items)) => { + let mut out = Vec::with_capacity(items.len()); + for item in items { + let role = item + .as_object() + .ok_or_else(|| strict("member role must be an object"))?; + // Reject extra keys: the TS parser passes role objects through + // verbatim, so any shape beyond {id, name} must use the TS path. + if role.keys().any(|k| k != "id" && k != "name") { + return Err(strict("member role has unsupported keys")); + } + out.push(RoleOut { + id: require_str(role, "id", "member role")?, + name: optional_str(role, "name", "member role")?, + }); + } + Some(out) + } + other => { + return Err(strict(format!( + "member.roles must be an array when present, got {}", + type_name(other) + ))) + } + }; + + Ok(ChatlabMemberOut { + platform_id: require_str(obj, "platformId", "member")?, + account_name: require_str(obj, "accountName", "member")?, + group_nickname: optional_str(obj, "groupNickname", "member")?, + avatar: optional_str(obj, "avatar", "member")?, + roles, + }) +} + +/// Tracks members collected from messages: first-seen order, whole entry +/// replaced on every message from the same sender (JS `Map.set` semantics). +struct CollectedMembers { + order: Vec<(String, String, Option)>, + index: HashMap, +} + +impl CollectedMembers { + fn new() -> Self { + CollectedMembers { + order: Vec::new(), + index: HashMap::new(), + } + } + + fn observe(&mut self, platform_id: &str, account_name: &str, group_nickname: Option<&String>) { + match self.index.get(platform_id) { + Some(&i) => { + self.order[i].1 = account_name.to_string(); + self.order[i].2 = group_nickname.cloned(); + } + None => { + self.index.insert(platform_id.to_string(), self.order.len()); + self.order.push(( + platform_id.to_string(), + account_name.to_string(), + group_nickname.cloned(), + )); + } + } + } +} + +pub fn parse_chatlab( + buf: &[u8], + file_path: &str, + mut on_progress: impl FnMut(u64, u64), +) -> ScanResult { + let mut meta_raw: Option<&[u8]> = None; + let mut members_raw: Option<&[u8]> = None; + let mut messages_raw: Option<&[u8]> = None; + + walk_top_level(buf, |key, raw| { + match key { + b"meta" => meta_raw = Some(raw), + b"members" => members_raw = Some(raw), + b"messages" => messages_raw = Some(raw), + _ => {} + } + Ok(()) + })?; + + let meta = parse_meta(meta_raw, file_path)?; + + let mut head_members: Vec = Vec::new(); + if let Some(raw) = members_raw { + let base_offset = raw.as_ptr() as usize - buf.as_ptr() as usize; + for_each_array_element(raw, base_offset, |element, _| { + head_members.push(parse_member(element)?); + Ok(()) + })?; + } + let members_from_head = !head_members.is_empty(); + + let mut collected = CollectedMembers::new(); + let mut messages: Vec = Vec::new(); + + if let Some(raw) = messages_raw { + let base_offset = raw.as_ptr() as usize - buf.as_ptr() as usize; + for_each_array_element(raw, base_offset, |element, end_offset| { + let value: Value = serde_json::from_slice(element) + .map_err(|err| strict(format!("invalid message: {err}")))?; + let obj = value + .as_object() + .ok_or_else(|| strict("message must be an object"))?; + + let sender = require_str(obj, "sender", "message")?; + let account_name = require_str(obj, "accountName", "message")?; + let group_nickname = optional_str(obj, "groupNickname", "message")?; + + let timestamp = match obj.get("timestamp") { + Some(Value::Number(n)) => n + .as_f64() + .ok_or_else(|| strict("message.timestamp out of range"))?, + other => { + return Err(strict(format!( + "message.timestamp must be a number, got {}", + type_name(other) + ))) + } + }; + let message_type = match obj.get("type") { + Some(Value::Number(n)) => n + .as_u64() + .and_then(|v| u32::try_from(v).ok()) + .ok_or_else(|| strict("message.type must be a non-negative integer"))?, + other => { + return Err(strict(format!( + "message.type must be a number, got {}", + type_name(other) + ))) + } + }; + let content = match obj.get("content") { + Some(Value::String(s)) => Some(s.clone()), + Some(Value::Null) => None, + other => { + return Err(strict(format!( + "message.content must be a string or null, got {}", + type_name(other) + ))) + } + }; + + if !members_from_head { + collected.observe(&sender, &account_name, group_nickname.as_ref()); + } + + messages.push(ChatlabMessageOut { + platform_message_id: optional_str(obj, "platformMessageId", "message")?, + sender_platform_id: sender, + sender_account_name: account_name, + sender_group_nickname: group_nickname, + timestamp, + message_type, + content, + reply_to_message_id: optional_str(obj, "replyToMessageId", "message")?, + }); + + on_progress(end_offset as u64, messages.len() as u64); + Ok(()) + })?; + } + + let members = if members_from_head { + head_members + } else { + collected + .order + .into_iter() + .map( + |(platform_id, account_name, group_nickname)| ChatlabMemberOut { + platform_id, + account_name, + group_nickname, + avatar: None, + roles: None, + }, + ) + .collect() + }; + + Ok(ChatlabOutput { + name: meta.name, + chat_type: meta.chat_type, + platform: meta.platform, + group_id: meta.group_id, + group_avatar: meta.group_avatar, + members_from_head, + members, + messages, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(doc: &str) -> ScanResult { + parse_chatlab(doc.as_bytes(), "/tmp/示例群.json", |_, _| {}) + } + + #[test] + fn parses_full_document() { + let doc = r#"{ + "chatlab": {"version": "0.0.2", "exportedAt": 1700000000}, + "meta": {"name": "测试群", "platform": "weixin", "type": "group", "groupId": "g1@chatroom", "groupAvatar": "https://x/y.jpg"}, + "members": [ + {"platformId": "u1", "accountName": "Alice", "groupNickname": "小A", "avatar": "https://a", "roles": [{"id": "owner"}, {"id": "admin", "name": "管理"}]}, + {"platformId": "u2", "accountName": "Bob"} + ], + "messages": [ + {"platformMessageId": "m1", "sender": "u1", "accountName": "Alice", "groupNickname": "小A", "timestamp": 100, "type": 0, "content": "hi"}, + {"platformMessageId": "m2", "sender": "u2", "accountName": "Bob", "timestamp": 101, "type": 25, "content": "reply", "replyToMessageId": "m1"}, + {"sender": "u2", "accountName": "Bob", "timestamp": 102, "type": 1, "content": null} + ] + }"#; + let out = parse(doc).expect("should parse"); + assert_eq!(out.name, "测试群"); + assert_eq!(out.platform, "weixin"); + assert_eq!(out.chat_type, "group"); + assert_eq!(out.group_id.as_deref(), Some("g1@chatroom")); + assert!(out.members_from_head); + assert_eq!(out.members.len(), 2); + let roles = out.members[0].roles.as_ref().unwrap(); + assert_eq!(roles[0].id, "owner"); + assert_eq!(roles[0].name, None); + assert_eq!(roles[1].name.as_deref(), Some("管理")); + assert!(out.members[1].roles.is_none()); + + assert_eq!(out.messages.len(), 3); + assert_eq!(out.messages[0].platform_message_id.as_deref(), Some("m1")); + assert_eq!(out.messages[1].reply_to_message_id.as_deref(), Some("m1")); + assert_eq!(out.messages[2].content, None); + assert_eq!(out.messages[2].platform_message_id, None); + } + + #[test] + fn collects_members_from_messages_when_head_absent_or_empty() { + for members_part in [r#""members": [],"#, ""] { + let doc = format!( + r#"{{"chatlab": {{"version": "1"}}, "meta": {{"name": "N"}}, {members_part} "messages": [ + {{"sender": "u1", "accountName": "A1", "groupNickname": "g", "timestamp": 1, "type": 0, "content": "x"}}, + {{"sender": "u1", "accountName": "A2", "timestamp": 2, "type": 0, "content": "y"}}, + {{"sender": "u2", "accountName": "B", "timestamp": 3, "type": 0, "content": "z"}} + ]}}"# + ); + let out = parse(&doc).expect("should parse"); + assert!(!out.members_from_head); + assert_eq!(out.members.len(), 2); + // Whole entry replaced by the later message (JS Map.set semantics): + // groupNickname reset to None even though the first message had one. + assert_eq!(out.members[0].account_name, "A2"); + assert_eq!(out.members[0].group_nickname, None); + } + } + + #[test] + fn meta_defaults_and_filename_fallback() { + let out = parse(r#"{"meta": {}, "messages": []}"#).expect("should parse"); + assert_eq!(out.name, "示例群"); + assert_eq!(out.platform, "unknown"); + assert_eq!(out.chat_type, "group"); + assert_eq!(out.group_id, None); + + // Explicit default name also falls back to the filename. + let out2 = parse(r#"{"meta": {"name": "未知群聊", "platform": "qq"}, "messages": []}"#) + .expect("should parse"); + assert_eq!(out2.name, "示例群"); + assert_eq!(out2.platform, "qq"); + + // Missing meta -> strict error (TS emits a differently-shaped default + // meta object, so that path must go through the fallback). + assert!(parse(r#"{"chatlab": {"version": "1"}, "messages": []}"#).is_err()); + } + + #[test] + fn strict_errors_on_off_spec_fields() { + // These must all error so the TS fallback (which replicates JS + // passthrough quirks) handles them instead. + let cases = [ + r#"{"messages": [{"sender": 5, "accountName": "A", "timestamp": 1, "type": 0, "content": "x"}]}"#, + r#"{"messages": [{"sender": "u", "accountName": "A", "timestamp": "1", "type": 0, "content": "x"}]}"#, + r#"{"messages": [{"sender": "u", "accountName": "A", "timestamp": 1, "type": 0.5, "content": "x"}]}"#, + r#"{"messages": [{"sender": "u", "accountName": "A", "timestamp": 1, "type": 0, "content": 42}]}"#, + r#"{"messages": [{"sender": "u", "accountName": "A", "timestamp": 1, "type": 0}]}"#, + r#"{"messages": [{"sender": "u", "accountName": "A", "timestamp": 1, "type": 0, "content": "x", "platformMessageId": 7}]}"#, + r#"{"members": [{"platformId": "u", "accountName": "A", "roles": [{"id": "owner", "extra": 1}]}], "messages": []}"#, + r#"{"members": [{"platformId": "u"}], "messages": []}"#, + r#"{"meta": {"name": 42}, "messages": []}"#, + r#"{"meta": {"groupId": null}, "messages": []}"#, + ]; + for doc in cases { + assert!(parse(doc).is_err(), "expected strict error for: {doc}"); + } + } + + #[test] + fn empty_strings_pass_through() { + let doc = r#"{"meta": {"name": "", "type": "", "groupId": ""}, "messages": [ + {"sender": "", "accountName": "", "timestamp": 1, "type": 0, "content": ""} + ]}"#; + let out = parse(doc).expect("should parse"); + // Empty name is falsy -> default -> filename fallback; empty type -> group. + assert_eq!(out.name, "示例群"); + assert_eq!(out.chat_type, "group"); + // groupId passes through verbatim, including empty string. + assert_eq!(out.group_id.as_deref(), Some("")); + assert_eq!(out.messages[0].sender_platform_id, ""); + assert_eq!(out.messages[0].content.as_deref(), Some("")); + } +} diff --git a/packages/parser-native/src/jsutil.rs b/packages/parser-native/src/jsutil.rs new file mode 100644 index 00000000..b3a8ccae --- /dev/null +++ b/packages/parser-native/src/jsutil.rs @@ -0,0 +1,32 @@ +//! Small helpers replicating JavaScript semantics shared by format kernels. + +use std::path::Path; + +use serde_json::Value; + +/// `path.basename(filePath).replace(/\.json$/i, '') || fallback` +pub fn extract_name_from_file_path(file_path: &str, fallback: &str) -> String { + let basename = Path::new(file_path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + let stripped = + if basename.len() >= 5 && basename[basename.len() - 5..].eq_ignore_ascii_case(".json") { + &basename[..basename.len() - 5] + } else { + basename.as_str() + }; + if stripped.is_empty() { + fallback.to_string() + } else { + stripped.to_string() + } +} + +/// `value || fallback` semantics for string fields: empty string is falsy. +pub fn non_empty_str(value: Option<&Value>) -> Option<&str> { + match value { + Some(Value::String(s)) if !s.is_empty() => Some(s.as_str()), + _ => None, + } +} diff --git a/packages/parser-native/src/lib.rs b/packages/parser-native/src/lib.rs new file mode 100644 index 00000000..d64c26c2 --- /dev/null +++ b/packages/parser-native/src/lib.rs @@ -0,0 +1,442 @@ +#![deny(clippy::all)] + +mod chatlab; +mod jsutil; +mod scanner; +mod weflow; + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; + +use napi::bindgen_prelude::*; +use napi_derive::napi; + +use chatlab::ChatlabOutput; +use weflow::{MessageOut, WeflowOutput}; + +#[napi(object)] +pub struct NativeParseProgress { + pub bytes_read: f64, + pub total_bytes: f64, + pub messages_processed: f64, +} + +// ==================== Shared parser state ==================== + +struct ParsedState { + output: T, + cursor: usize, +} + +struct Shared { + bytes_read: AtomicU64, + total_bytes: AtomicU64, + messages_processed: AtomicU64, + parsed: Mutex>>, +} + +impl Shared { + fn new() -> Arc { + Arc::new(Shared { + bytes_read: AtomicU64::new(0), + total_bytes: AtomicU64::new(0), + messages_processed: AtomicU64::new(0), + parsed: Mutex::new(None), + }) + } + + fn progress(&self) -> NativeParseProgress { + NativeParseProgress { + bytes_read: self.bytes_read.load(Ordering::Relaxed) as f64, + total_bytes: self.total_bytes.load(Ordering::Relaxed) as f64, + messages_processed: self.messages_processed.load(Ordering::Relaxed) as f64, + } + } + + fn lock_parsed(&self) -> Result>>> { + self.parsed + .lock() + .map_err(|_| Error::from_reason("parser state poisoned")) + } + + /// Run a format kernel on the file, tracking progress atomics. + fn run_parse( + &self, + file_path: &str, + kernel: impl Fn( + &[u8], + &str, + &mut dyn FnMut(u64, u64), + ) -> std::result::Result, + ) -> Result<()> { + let buf = std::fs::read(file_path) + .map_err(|err| Error::from_reason(format!("failed to read file: {err}")))?; + self.total_bytes.store(buf.len() as u64, Ordering::Relaxed); + + let mut on_progress = |bytes: u64, messages: u64| { + self.bytes_read.store(bytes, Ordering::Relaxed); + self.messages_processed.store(messages, Ordering::Relaxed); + }; + let output = kernel(&buf, file_path, &mut on_progress) + .map_err(|err| Error::from_reason(format!("parse failed: {err}")))?; + + self.bytes_read.store(buf.len() as u64, Ordering::Relaxed); + let mut parsed = self.lock_parsed()?; + *parsed = Some(ParsedState { output, cursor: 0 }); + Ok(()) + } +} + +/// Take the next `size` messages out of `messages`, advancing `cursor`. +fn take_batch_from( + state: &mut ParsedState, + messages_of: impl Fn(&mut O) -> &mut Vec, + size: u32, + map: impl Fn(M) -> M, +) -> Option> { + let cursor = state.cursor; + let messages = messages_of(&mut state.output); + let total = messages.len(); + if cursor >= total { + return None; + } + let end = (cursor + size.max(1) as usize).min(total); + let mut batch = Vec::with_capacity(end - cursor); + for message in &mut messages[cursor..end] { + batch.push(map(std::mem::take(message))); + } + state.cursor = end; + Some(batch) +} + +// ==================== WeFlow ==================== + +#[napi(object)] +pub struct NativeWeflowMeta { + pub name: String, + /// "group" | "private" (ChatType enum string values) + pub chat_type: String, + pub group_id: Option, + pub group_avatar: Option, + pub owner_id: Option, + pub message_count: f64, + pub member_count: f64, +} + +#[napi(object)] +pub struct NativeParsedMember { + pub platform_id: String, + pub account_name: String, + pub avatar: Option, +} + +#[napi(object)] +pub struct NativeParsedMessage { + pub platform_message_id: String, + pub sender_platform_id: String, + pub sender_account_name: String, + /// null when the source `createTime` was JSON null (importer skips those). + pub timestamp: Option, + /// Numeric MessageType enum value from shared-types. + pub message_type: u32, + pub content: Option, +} + +pub struct ParseWeflowTask { + file_path: String, + shared: Arc>, +} + +impl Task for ParseWeflowTask { + type Output = (); + type JsValue = (); + + fn compute(&mut self) -> Result { + self.shared + .run_parse(&self.file_path, |buf, path, on_progress| { + weflow::parse_weflow(buf, path, on_progress) + }) + } + + fn resolve(&mut self, _env: Env, _output: Self::Output) -> Result { + Ok(()) + } +} + +/// Rust WeFlow parser. Usage from JS: +/// `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. +#[napi] +pub struct WeflowParser { + file_path: String, + shared: Arc>, +} + +#[napi] +impl WeflowParser { + #[napi(constructor)] + pub fn new(file_path: String) -> Self { + WeflowParser { + file_path, + shared: Shared::new(), + } + } + + /// Run the full parse on the libuv thread pool. + #[napi] + pub fn parse(&self) -> AsyncTask { + AsyncTask::new(ParseWeflowTask { + file_path: self.file_path.clone(), + shared: Arc::clone(&self.shared), + }) + } + + /// Poll progress while `parse()` is pending. + #[napi] + pub fn progress(&self) -> NativeParseProgress { + self.shared.progress() + } + + #[napi] + pub fn get_meta(&self) -> Result { + let parsed = self.shared.lock_parsed()?; + let state = parsed + .as_ref() + .ok_or_else(|| Error::from_reason("getMeta() called before parse() completed"))?; + Ok(NativeWeflowMeta { + name: state.output.name.clone(), + chat_type: state.output.chat_type.to_string(), + group_id: state.output.group_id.clone(), + group_avatar: state.output.group_avatar.clone(), + owner_id: state.output.owner_id.clone(), + message_count: state.output.messages.len() as f64, + member_count: state.output.members.len() as f64, + }) + } + + /// Move members (with avatars) out to JS. Callable once. + #[napi] + pub fn take_members(&self) -> Result> { + let mut parsed = self.shared.lock_parsed()?; + let state = parsed + .as_mut() + .ok_or_else(|| Error::from_reason("takeMembers() called before parse() completed"))?; + Ok(std::mem::take(&mut state.output.members) + .into_iter() + .map(|m| NativeParsedMember { + platform_id: m.platform_id, + account_name: m.account_name, + avatar: m.avatar, + }) + .collect()) + } + + /// Move the next batch of messages out to JS; null when exhausted. + #[napi] + pub fn take_batch(&self, size: u32) -> Result>> { + let mut parsed = self.shared.lock_parsed()?; + let state = parsed + .as_mut() + .ok_or_else(|| Error::from_reason("takeBatch() called before parse() completed"))?; + Ok( + take_batch_from(state, |o| &mut o.messages, size, |message| message).map(|batch| { + batch + .into_iter() + .map(|message| { + let MessageOut { + platform_message_id, + sender_platform_id, + sender_account_name, + timestamp, + message_type, + content, + } = message; + NativeParsedMessage { + platform_message_id, + sender_platform_id, + sender_account_name, + timestamp, + message_type, + content, + } + }) + .collect() + }), + ) + } +} + +// ==================== ChatLab JSON ==================== + +#[napi(object)] +pub struct NativeChatlabMeta { + pub name: String, + /// Passthrough of `meta.type` ("group" when missing), matching the TS parser. + pub chat_type: String, + /// Passthrough of `meta.platform` ("unknown" when missing). + pub platform: String, + pub group_id: Option, + pub group_avatar: Option, + /// true when members came from the top-level `members` array; + /// false when collected from messages (TS emits different object shapes). + pub members_from_head: bool, + pub message_count: f64, + pub member_count: f64, +} + +#[napi(object)] +pub struct NativeMemberRole { + pub id: String, + /// Present only when the source role object had a `name` key. + pub name: Option, +} + +#[napi(object)] +pub struct NativeChatlabMember { + pub platform_id: String, + pub account_name: String, + pub group_nickname: Option, + pub avatar: Option, + pub roles: Option>, +} + +#[napi(object)] +#[derive(Default)] +pub struct NativeChatlabMessage { + pub platform_message_id: Option, + pub sender_platform_id: String, + pub sender_account_name: String, + pub sender_group_nickname: Option, + pub timestamp: f64, + /// Numeric MessageType enum value from shared-types. + pub message_type: u32, + /// undefined here means the source `content` was JSON null. + pub content: Option, + pub reply_to_message_id: Option, +} + +pub struct ParseChatlabTask { + file_path: String, + shared: Arc>, +} + +impl Task for ParseChatlabTask { + type Output = (); + type JsValue = (); + + fn compute(&mut self) -> Result { + self.shared + .run_parse(&self.file_path, |buf, path, on_progress| { + chatlab::parse_chatlab(buf, path, on_progress) + }) + } + + fn resolve(&mut self, _env: Env, _output: Self::Output) -> Result { + Ok(()) + } +} + +/// Rust ChatLab JSON parser. Strict about the format spec: off-spec files +/// error out and the JS side falls back to the pure-TS parser. +/// Usage from JS: `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. +#[napi] +pub struct ChatlabParser { + file_path: String, + shared: Arc>, +} + +#[napi] +impl ChatlabParser { + #[napi(constructor)] + pub fn new(file_path: String) -> Self { + ChatlabParser { + file_path, + shared: Shared::new(), + } + } + + /// Run the full parse on the libuv thread pool. + #[napi] + pub fn parse(&self) -> AsyncTask { + AsyncTask::new(ParseChatlabTask { + file_path: self.file_path.clone(), + shared: Arc::clone(&self.shared), + }) + } + + /// Poll progress while `parse()` is pending. + #[napi] + pub fn progress(&self) -> NativeParseProgress { + self.shared.progress() + } + + #[napi] + pub fn get_meta(&self) -> Result { + let parsed = self.shared.lock_parsed()?; + let state = parsed + .as_ref() + .ok_or_else(|| Error::from_reason("getMeta() called before parse() completed"))?; + Ok(NativeChatlabMeta { + name: state.output.name.clone(), + chat_type: state.output.chat_type.clone(), + platform: state.output.platform.clone(), + group_id: state.output.group_id.clone(), + group_avatar: state.output.group_avatar.clone(), + members_from_head: state.output.members_from_head, + message_count: state.output.messages.len() as f64, + member_count: state.output.members.len() as f64, + }) + } + + /// Move members out to JS. Callable once. + #[napi] + pub fn take_members(&self) -> Result> { + let mut parsed = self.shared.lock_parsed()?; + let state = parsed + .as_mut() + .ok_or_else(|| Error::from_reason("takeMembers() called before parse() completed"))?; + Ok(std::mem::take(&mut state.output.members) + .into_iter() + .map(|m| NativeChatlabMember { + platform_id: m.platform_id, + account_name: m.account_name, + group_nickname: m.group_nickname, + avatar: m.avatar, + roles: m.roles.map(|roles| { + roles + .into_iter() + .map(|r| NativeMemberRole { + id: r.id, + name: r.name, + }) + .collect() + }), + }) + .collect()) + } + + /// Move the next batch of messages out to JS; null when exhausted. + #[napi] + pub fn take_batch(&self, size: u32) -> Result>> { + let mut parsed = self.shared.lock_parsed()?; + let state = parsed + .as_mut() + .ok_or_else(|| Error::from_reason("takeBatch() called before parse() completed"))?; + Ok( + take_batch_from(state, |o| &mut o.messages, size, |m| m).map(|batch| { + batch + .into_iter() + .map(|message| NativeChatlabMessage { + platform_message_id: message.platform_message_id, + sender_platform_id: message.sender_platform_id, + sender_account_name: message.sender_account_name, + sender_group_nickname: message.sender_group_nickname, + timestamp: message.timestamp, + message_type: message.message_type, + content: message.content, + reply_to_message_id: message.reply_to_message_id, + }) + .collect() + }), + ) + } +} diff --git a/packages/parser-native/src/scanner.rs b/packages/parser-native/src/scanner.rs new file mode 100644 index 00000000..7db83c04 --- /dev/null +++ b/packages/parser-native/src/scanner.rs @@ -0,0 +1,331 @@ +//! Shared byte-level JSON scanning utilities. +//! +//! This is the format-agnostic "common layer": a structural scanner over a +//! UTF-8 JSON buffer that can locate top-level object keys and hand out raw +//! value spans without building a DOM for the whole file. Format kernels +//! (e.g. WeFlow) drive it and decide which spans to materialize with +//! serde_json. + +use memchr::memchr2; + +#[derive(Debug)] +pub struct ScanError { + pub message: String, + pub offset: usize, +} + +impl ScanError { + fn new(message: impl Into, offset: usize) -> Self { + ScanError { + message: message.into(), + offset, + } + } +} + +impl std::fmt::Display for ScanError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} (byte offset {})", self.message, self.offset) + } +} + +impl std::error::Error for ScanError {} + +pub type ScanResult = Result; + +pub struct JsonScanner<'a> { + buf: &'a [u8], + pos: usize, +} + +impl<'a> JsonScanner<'a> { + pub fn new(buf: &'a [u8]) -> Self { + let mut pos = 0; + // Skip UTF-8 BOM if present. + if buf.len() >= 3 && &buf[0..3] == b"\xEF\xBB\xBF" { + pos = 3; + } + JsonScanner { buf, pos } + } + + pub fn pos(&self) -> usize { + self.pos + } + + pub fn skip_ws(&mut self) { + while self.pos < self.buf.len() { + match self.buf[self.pos] { + b' ' | b'\t' | b'\n' | b'\r' => self.pos += 1, + _ => break, + } + } + } + + fn peek(&self) -> Option { + self.buf.get(self.pos).copied() + } + + fn peek_is(&self, byte: u8) -> bool { + self.peek() == Some(byte) + } + + pub fn expect(&mut self, byte: u8) -> ScanResult<()> { + match self.peek() { + Some(b) if b == byte => { + self.pos += 1; + Ok(()) + } + Some(b) => Err(ScanError::new( + format!("expected '{}' but found '{}'", byte as char, b as char), + self.pos, + )), + None => Err(ScanError::new( + format!("expected '{}' but reached end of input", byte as char), + self.pos, + )), + } + } + + pub fn consume(&mut self, byte: u8) -> bool { + if self.peek() == Some(byte) { + self.pos += 1; + true + } else { + false + } + } + + /// Scan a JSON string starting at the opening quote. + /// Returns the raw bytes between the quotes (escapes untouched). + pub fn scan_string(&mut self) -> ScanResult<&'a [u8]> { + self.expect(b'"')?; + let start = self.pos; + loop { + match memchr2(b'"', b'\\', &self.buf[self.pos..]) { + Some(offset) => { + let at = self.pos + offset; + if self.buf[at] == b'"' { + self.pos = at + 1; + return Ok(&self.buf[start..at]); + } + // Escape sequence: skip backslash + escaped byte. + if at + 1 >= self.buf.len() { + return Err(ScanError::new("unterminated escape sequence", at)); + } + self.pos = at + 2; + } + None => return Err(ScanError::new("unterminated string", start)), + } + } + } + + /// Scan any JSON value and return its full raw span. + pub fn scan_value(&mut self) -> ScanResult<&'a [u8]> { + self.skip_ws(); + let start = self.pos; + match self.peek() { + Some(b'"') => { + self.scan_string()?; + } + Some(b'{') | Some(b'[') => { + self.scan_container()?; + } + Some(_) => { + // Scalar: number / true / false / null. Scan until a structural + // delimiter or whitespace. + while self.pos < self.buf.len() { + match self.buf[self.pos] { + b',' | b'}' | b']' | b' ' | b'\t' | b'\n' | b'\r' => break, + _ => self.pos += 1, + } + } + if self.pos == start { + return Err(ScanError::new("expected a JSON value", start)); + } + } + None => { + return Err(ScanError::new( + "expected a JSON value but reached end of input", + start, + )) + } + } + Ok(&self.buf[start..self.pos]) + } + + /// Scan a `{...}` or `[...]` container assuming the opener is at `pos`. + /// + /// Structural bytes are walked one at a time (containers are mostly small), + /// while string contents — the bulk of large payloads such as base64 + /// avatars — are skipped via the memchr fast path in `scan_string`. + fn scan_container(&mut self) -> ScanResult<()> { + let start = self.pos; + let mut depth: usize = 0; + while self.pos < self.buf.len() { + match self.buf[self.pos] { + b'"' => { + self.scan_string()?; + } + b'{' | b'[' => { + depth += 1; + self.pos += 1; + } + b'}' | b']' => { + depth -= 1; + self.pos += 1; + if depth == 0 { + return Ok(()); + } + } + _ => self.pos += 1, + } + } + Err(ScanError::new("unterminated container", start)) + } +} + +/// Walk the top-level JSON object and hand each key's raw value span to the +/// callback. Key order in the document does not matter to callers because +/// they receive every entry before acting on relationships between keys. +pub fn walk_top_level<'a>( + buf: &'a [u8], + mut on_entry: impl FnMut(&'a [u8], &'a [u8]) -> ScanResult<()>, +) -> ScanResult<()> { + let mut scanner = JsonScanner::new(buf); + scanner.skip_ws(); + scanner.expect(b'{')?; + scanner.skip_ws(); + if scanner.consume(b'}') { + return Ok(()); + } + + loop { + scanner.skip_ws(); + let key = scanner.scan_string()?; + scanner.skip_ws(); + scanner.expect(b':')?; + let raw = scanner.scan_value()?; + on_entry(key, raw)?; + + scanner.skip_ws(); + if scanner.consume(b',') { + continue; + } + scanner.expect(b'}')?; + return Ok(()); + } +} + +/// Iterate the elements of a raw JSON array span (as produced by +/// `walk_top_level`). The callback receives each element's raw span plus the +/// absolute end offset of that element relative to `base_offset` — used for +/// progress reporting on huge arrays. +pub fn for_each_array_element<'a>( + raw: &'a [u8], + base_offset: usize, + mut on_element: impl FnMut(&'a [u8], usize) -> ScanResult<()>, +) -> ScanResult<()> { + let mut scanner = JsonScanner::new(raw); + scanner.skip_ws(); + if !scanner.peek_is(b'[') { + return Err(ScanError::new("expected an array", scanner.pos())); + } + scanner.expect(b'[')?; + scanner.skip_ws(); + if scanner.consume(b']') { + return Ok(()); + } + + loop { + let element = scanner.scan_value()?; + on_element(element, base_offset + scanner.pos())?; + scanner.skip_ws(); + if scanner.consume(b',') { + scanner.skip_ws(); + continue; + } + scanner.expect(b']')?; + return Ok(()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect_entries(doc: &str) -> Vec<(String, String)> { + let mut entries = Vec::new(); + walk_top_level(doc.as_bytes(), |key, raw| { + entries.push(( + String::from_utf8_lossy(key).to_string(), + String::from_utf8_lossy(raw).to_string(), + )); + Ok(()) + }) + .expect("walk should succeed"); + entries + } + + fn collect_elements(raw: &str) -> Vec { + let mut elements = Vec::new(); + for_each_array_element(raw.as_bytes(), 0, |element, _| { + elements.push(String::from_utf8_lossy(element).to_string()); + Ok(()) + }) + .expect("iteration should succeed"); + elements + } + + #[test] + fn walks_simple_object() { + let entries = collect_entries(r#"{"a": 1, "b": "x", "c": [1, 2]}"#); + assert_eq!( + entries, + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), "\"x\"".to_string()), + ("c".to_string(), "[1, 2]".to_string()), + ] + ); + } + + #[test] + fn handles_nested_and_escaped_content() { + let doc = r#"{"session": {"name": "a\"}b", "inner": {"x": [1, {"y": "]"}]}}, "messages": [{"content": "hi \\ {there}"}, {"content": "\u4f60好"}]}"#; + let entries = collect_entries(doc); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].0, "session"); + assert!(entries[0].1.starts_with('{') && entries[0].1.ends_with('}')); + + let elements = collect_elements(&entries[1].1); + assert_eq!(elements.len(), 2); + assert_eq!(elements[0], r#"{"content": "hi \\ {there}"}"#); + } + + #[test] + fn handles_empty_arrays_and_objects() { + let entries = collect_entries(r#"{"avatars": {}, "messages": []}"#); + assert_eq!( + entries, + vec![ + ("avatars".to_string(), "{}".to_string()), + ("messages".to_string(), "[]".to_string()), + ] + ); + assert!(collect_elements("[]").is_empty()); + } + + #[test] + fn skips_bom() { + let doc = "\u{FEFF}{\"messages\": [3]}"; + let entries = collect_entries(doc); + assert_eq!(entries[0].0, "messages"); + assert_eq!(collect_elements(&entries[0].1), vec!["3".to_string()]); + } + + #[test] + fn errors_on_truncated_document() { + let result = walk_top_level(br#"{"a": [1, 2"#, |_, _| Ok(())); + assert!(result.is_err()); + } +} diff --git a/packages/parser-native/src/weflow.rs b/packages/parser-native/src/weflow.rs new file mode 100644 index 00000000..0a70923f --- /dev/null +++ b/packages/parser-native/src/weflow.rs @@ -0,0 +1,432 @@ +//! WeFlow (WeChat export) format kernel. +//! +//! Replicates the behavior of `packages/parser/src/formats/weflow.ts` in a +//! single pass over the file buffer. The TS implementation is the reference: +//! any semantic difference is a bug (covered by TS-side parity tests). + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::jsutil::{extract_name_from_file_path, non_empty_str}; +use crate::scanner::{for_each_array_element, walk_top_level, ScanResult}; + +pub struct MemberOut { + pub platform_id: String, + pub account_name: String, + pub avatar: Option, +} + +#[derive(Default)] +pub struct MessageOut { + pub platform_message_id: String, + pub sender_platform_id: String, + pub sender_account_name: String, + pub timestamp: Option, + pub message_type: u32, + pub content: Option, +} + +pub struct WeflowOutput { + pub name: String, + /// "group" | "private" — matches the ChatType enum string values. + pub chat_type: &'static str, + pub group_id: Option, + pub group_avatar: Option, + pub owner_id: Option, + pub members: Vec, + pub messages: Vec, +} + +/// Message type mapping, mirroring `convertMessageType` in weflow.ts. +/// Values are the numeric MessageType enum members from shared-types. +fn convert_message_type(type_str: Option<&str>) -> u32 { + match type_str { + Some("文本消息") => 0, // TEXT + Some("图片消息") => 1, // IMAGE + Some("语音消息") => 2, // VOICE + Some("视频消息") => 3, // VIDEO + Some("文件消息") => 4, // FILE + Some("动画表情") => 5, // EMOJI + Some("名片消息") => 27, // CONTACT + Some("卡片式链接") | Some("图文消息") => 7, // LINK + Some("位置消息") => 8, // LOCATION + Some("红包卡片") => 20, // RED_PACKET + Some("转账卡片") => 21, // TRANSFER + Some("小程序分享") | Some("视频号直播卡片") => 24, // SHARE + Some("引用消息") => 25, // REPLY + Some("聊天记录合并转发") => 26, // FORWARD + Some("系统消息") => 80, // SYSTEM + _ => 99, // OTHER + } +} + +/// JavaScript `String.prototype.trim` semantics: Unicode White_Space minus +/// U+0085 (NEL, not trimmed by JS), plus U+FEFF (trimmed by JS). +fn js_trim(input: &str) -> &str { + input.trim_matches(|c: char| (c.is_whitespace() && c != '\u{85}') || c == '\u{FEFF}') +} + +/// JS `String(x)` semantics for the localId field. +fn js_string(value: Option<&Value>) -> String { + match value { + None => "undefined".to_string(), + Some(Value::Null) => "null".to_string(), + Some(Value::String(s)) => s.clone(), + Some(Value::Bool(b)) => b.to_string(), + Some(Value::Number(n)) => n.to_string(), + // Arrays/objects are pathological for an ID field; compact JSON is a + // reasonable stand-in for JS coercion here. + Some(other) => serde_json::to_string(other).unwrap_or_default(), + } +} + +fn is_send_equals_one(value: Option<&Value>) -> bool { + matches!(value, Some(Value::Number(n)) if n.as_f64() == Some(1.0)) +} + +struct MemberTracker { + order: Vec, + index: HashMap, +} + +impl MemberTracker { + fn new() -> Self { + MemberTracker { + order: Vec::new(), + index: HashMap::new(), + } + } + + fn observe(&mut self, platform_id: &str, account_name: &str, avatar: Option<&String>) { + match self.index.get(platform_id) { + Some(&i) => { + let member = &mut self.order[i]; + // Latest display name wins; avatar only overwritten when present. + member.account_name = account_name.to_string(); + if let Some(a) = avatar { + member.avatar = Some(a.clone()); + } + } + None => { + self.index.insert(platform_id.to_string(), self.order.len()); + self.order.push(MemberOut { + platform_id: platform_id.to_string(), + account_name: account_name.to_string(), + avatar: avatar.cloned(), + }); + } + } + } +} + +pub fn parse_weflow( + buf: &[u8], + file_path: &str, + mut on_progress: impl FnMut(u64, u64), +) -> ScanResult { + // Pass 1: structural walk of the top-level object to locate the raw spans + // we care about. `scan_value` skims containers at memchr speed, so this is + // cheap even when `messages` is hundreds of MB. + let mut session_raw: Option<&[u8]> = None; + let mut avatars_raw: Option<&[u8]> = None; + let mut messages_raw: Option<&[u8]> = None; + + walk_top_level(buf, |key, raw| { + match key { + b"session" => session_raw = Some(raw), + b"avatars" => avatars_raw = Some(raw), + b"messages" => messages_raw = Some(raw), + _ => {} + } + Ok(()) + })?; + + // session: parse failure or non-object behaves like "no session" (the TS + // code catches parse errors and reads fields off a possibly-null object). + let session: Option> = session_raw + .and_then(|raw| serde_json::from_slice::(raw).ok()) + .and_then(|v| match v { + Value::Object(map) => Some(map), + _ => None, + }); + + // avatars: wxid -> base64 data URL; only non-empty strings are kept. + let mut avatars: HashMap = HashMap::new(); + if let Some(raw) = avatars_raw { + if let Ok(Value::Object(map)) = serde_json::from_slice::(raw) { + for (wxid, v) in map { + if let Value::String(s) = v { + if !s.is_empty() { + avatars.insert(wxid, s); + } + } + } + } + } + + // Chat type: '私聊' -> private, '群聊' -> group, otherwise infer from wxid. + let session_type = session.as_ref().and_then(|s| non_empty_str(s.get("type"))); + let session_wxid = session.as_ref().and_then(|s| non_empty_str(s.get("wxid"))); + let chat_type: &'static str = match session_type { + Some("私聊") => "private", + Some("群聊") => "group", + _ => match session_wxid { + Some(wxid) if !wxid.ends_with("@chatroom") => "private", + _ => "group", + }, + }; + + let name = session + .as_ref() + .and_then(|s| non_empty_str(s.get("displayName"))) + .or_else(|| { + session + .as_ref() + .and_then(|s| non_empty_str(s.get("nickname"))) + }) + .map(|s| s.to_string()) + .unwrap_or_else(|| extract_name_from_file_path(file_path, "未知聊天")); + + let group_id: Option = if chat_type == "group" { + session_wxid.map(|s| s.to_string()) + } else { + None + }; + + let group_avatar: Option = session + .as_ref() + .and_then(|s| non_empty_str(s.get("avatar"))) + .map(|s| s.to_string()) + .or_else(|| group_id.as_ref().and_then(|id| avatars.get(id).cloned())); + + // Pass 2: stream the messages array element by element. + let mut members = MemberTracker::new(); + let mut messages: Vec = Vec::new(); + let mut owner_id: Option = None; + + if let Some(raw) = messages_raw { + let base_offset = raw.as_ptr() as usize - buf.as_ptr() as usize; + for_each_array_element(raw, base_offset, |element, end_offset| { + let value: Value = match serde_json::from_slice(element) { + Ok(v) => v, + Err(_) => { + // Structurally valid JSON always deserializes into Value; this arm + // is unreachable in practice but must not abort the whole import. + return Ok(()); + } + }; + let obj = match value.as_object() { + Some(o) => o, + None => return Ok(()), + }; + + // ownerId: first message sent by self (isSend === 1) from a non-room id. + if owner_id.is_none() && is_send_equals_one(obj.get("isSend")) { + if let Some(username) = non_empty_str(obj.get("senderUsername")) { + if !username.ends_with("@chatroom") { + owner_id = Some(username.to_string()); + } + } + } + + // processMessage() replication. + let sender = match non_empty_str(obj.get("senderUsername")) { + Some(s) => s, + None => return Ok(()), + }; + if !obj.contains_key("createTime") { + return Ok(()); + } + if sender.ends_with("@chatroom") { + return Ok(()); + } + + let account_name = non_empty_str(obj.get("senderDisplayName")).unwrap_or(sender); + + // avatarKey = senderAvatarKey || senderUsername; a non-string truthy + // key can never hit the (string-keyed) avatars map, mirroring JS. + let avatar = match obj.get("senderAvatarKey") { + Some(Value::String(s)) if !s.is_empty() => avatars.get(s.as_str()), + None | Some(Value::Null) => avatars.get(sender), + Some(Value::String(_)) => avatars.get(sender), // empty string is falsy + Some(Value::Bool(false)) => avatars.get(sender), + Some(Value::Number(n)) if n.as_f64() == Some(0.0) => avatars.get(sender), + Some(_) => None, + }; + + members.observe(sender, account_name, avatar); + + let timestamp = match obj.get("createTime") { + Some(Value::Number(n)) => n.as_f64(), + _ => None, + }; + + let content: Option = match obj.get("content") { + None | Some(Value::Null) => None, + Some(Value::String(s)) => { + let trimmed = js_trim(s); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } + } + Some(other) => { + let serialized = serde_json::to_string(other).unwrap_or_default(); + let trimmed = js_trim(&serialized); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } + } + }; + + messages.push(MessageOut { + platform_message_id: js_string(obj.get("localId")), + sender_platform_id: sender.to_string(), + sender_account_name: account_name.to_string(), + timestamp, + message_type: convert_message_type(obj.get("type").and_then(|v| v.as_str())), + content, + }); + + on_progress(end_offset as u64, messages.len() as u64); + Ok(()) + })?; + } + + Ok(WeflowOutput { + name, + chat_type, + group_id, + group_avatar, + owner_id, + members: members.order, + messages, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(doc: &str) -> WeflowOutput { + parse_weflow(doc.as_bytes(), "/tmp/测试聊天.json", |_, _| {}).expect("parse should succeed") + } + + #[test] + fn parses_basic_group_export() { + let doc = r#"{ + "weflow": {"version": "1.0.0"}, + "session": {"wxid": "room@chatroom", "nickname": "群名", "displayName": "群显示名", "type": "群聊", "avatar": ""}, + "avatars": {"wxid_a": "data:image/jpeg;base64,AAA", "room@chatroom": "data:image/jpeg;base64,ROOM", "wxid_empty": ""}, + "messages": [ + {"localId": 1, "createTime": 100, "type": "文本消息", "content": " hello ", "isSend": 0, "senderUsername": "wxid_a", "senderDisplayName": "Alice", "senderAvatarKey": "wxid_a"}, + {"localId": 2, "createTime": 101, "type": "图片消息", "content": "[图片]", "isSend": 1, "senderUsername": "wxid_b", "senderDisplayName": "Bob"}, + {"localId": 3, "createTime": 102, "type": "系统消息", "content": "sys", "isSend": null, "senderUsername": "room@chatroom", "senderDisplayName": "群"}, + {"localId": 4, "createTime": 103, "type": "未知类型(88)", "content": null, "isSend": 0, "senderUsername": "wxid_a", "senderDisplayName": "Alice2"} + ] + }"#; + let out = parse(doc); + assert_eq!(out.name, "群显示名"); + assert_eq!(out.chat_type, "group"); + assert_eq!(out.group_id.as_deref(), Some("room@chatroom")); + // session.avatar is empty (falsy) -> falls back to avatars[groupId]. + assert_eq!( + out.group_avatar.as_deref(), + Some("data:image/jpeg;base64,ROOM") + ); + assert_eq!(out.owner_id.as_deref(), Some("wxid_b")); + + // room@chatroom sender is skipped entirely. + assert_eq!(out.messages.len(), 3); + assert_eq!(out.messages[0].content.as_deref(), Some("hello")); + assert_eq!(out.messages[0].message_type, 0); + assert_eq!(out.messages[0].platform_message_id, "1"); + assert_eq!(out.messages[1].message_type, 1); + assert_eq!(out.messages[2].message_type, 99); + assert_eq!(out.messages[2].content, None); + + // Members in first-seen order; Alice's name updated by her later message. + assert_eq!(out.members.len(), 2); + assert_eq!(out.members[0].platform_id, "wxid_a"); + assert_eq!(out.members[0].account_name, "Alice2"); + assert_eq!( + out.members[0].avatar.as_deref(), + Some("data:image/jpeg;base64,AAA") + ); + assert_eq!(out.members[1].platform_id, "wxid_b"); + assert_eq!(out.members[1].avatar, None); + } + + #[test] + fn message_filtering_matches_ts() { + let doc = r#"{ + "messages": [ + {"createTime": 1, "type": "文本消息", "content": "no sender"}, + {"senderUsername": "", "createTime": 1, "content": "empty sender"}, + {"senderUsername": "wxid_a", "content": "no createTime"}, + {"senderUsername": "wxid_a", "createTime": null, "content": "null createTime"}, + {"senderUsername": "wxid_a", "createTime": 5, "senderDisplayName": "", "content": " "} + ] + }"#; + let out = parse(doc); + assert_eq!(out.messages.len(), 2); + // null createTime passes the parser filter with a null timestamp. + assert_eq!(out.messages[0].timestamp, None); + assert_eq!(out.messages[0].content.as_deref(), Some("null createTime")); + // Empty displayName falls back to the platform id. + assert_eq!(out.messages[1].sender_account_name, "wxid_a"); + // Whitespace-only content is normalized to null. + assert_eq!(out.messages[1].content, None); + // localId missing -> String(undefined). + assert_eq!(out.messages[0].platform_message_id, "undefined"); + } + + #[test] + fn non_string_content_is_stringified() { + let doc = r#"{"messages": [ + {"senderUsername": "a", "createTime": 1, "content": {"k": 1, "a": [true, null]}}, + {"senderUsername": "a", "createTime": 2, "content": 42} + ]}"#; + let out = parse(doc); + assert_eq!( + out.messages[0].content.as_deref(), + Some(r#"{"k":1,"a":[true,null]}"#) + ); + assert_eq!(out.messages[1].content.as_deref(), Some("42")); + } + + #[test] + fn private_chat_and_filename_fallback() { + let doc = r#"{"session": {"wxid": "wxid_friend", "type": "其他"}, "messages": []}"#; + let out = parse(doc); + assert_eq!(out.chat_type, "private"); + assert_eq!(out.group_id, None); + assert_eq!(out.name, "测试聊天"); + + let out2 = parse(r#"{"messages": []}"#); + assert_eq!(out2.chat_type, "group"); + assert_eq!(out2.name, "测试聊天"); + } + + #[test] + fn js_trim_matches_javascript() { + assert_eq!(js_trim("\u{FEFF} hi \u{00A0}"), "hi"); + // U+0085 NEL is NOT trimmed by JS. + assert_eq!(js_trim("\u{85}x"), "\u{85}x"); + assert_eq!(js_trim("\n\t x \r\n"), "x"); + } + + #[test] + fn js_string_semantics_for_local_id() { + let n: Value = serde_json::from_str("3.5").unwrap(); + assert_eq!(js_string(Some(&n)), "3.5"); + assert_eq!(js_string(Some(&Value::Null)), "null"); + assert_eq!(js_string(None), "undefined"); + assert_eq!(js_string(Some(&Value::String("abc".into()))), "abc"); + assert_eq!(js_string(Some(&Value::Bool(true))), "true"); + } +} diff --git a/packages/parser/package.json b/packages/parser/package.json index 8cf73896..ee75ea2f 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -7,6 +7,7 @@ "main": "./src/index.ts", "types": "./src/index.ts", "dependencies": { + "@openchatlab/parser-native": "workspace:*", "@openchatlab/shared-types": "workspace:*", "stream-json": "^1.9.1" }, diff --git a/packages/parser/src/formats/chatlab.ts b/packages/parser/src/formats/chatlab.ts index 65ad03b2..ac4aca7d 100644 --- a/packages/parser/src/formats/chatlab.ts +++ b/packages/parser/src/formats/chatlab.ts @@ -276,9 +276,15 @@ async function* parseChatLab(options: ParseOptions): AsyncGenerator { + const result: ParseResult = { + meta: { name: '', platform: '', type: 'group' as ParseResult['meta']['type'] }, + members: [], + messages: [], + } + for await (const event of parseFile({ filePath, onLog: (_level, message) => logs.push(message) })) { + if (event.type === 'meta') result.meta = event.data + else if (event.type === 'members') result.members.push(...event.data) + else if (event.type === 'messages') result.messages.push(...event.data) + else if (event.type === 'error') throw event.data + } + return result +} + +const ENV_KEY = 'CHATLAB_DISABLE_NATIVE_PERF' + +function nativeAvailable(): boolean { + const saved = process.env[ENV_KEY] + delete process.env[ENV_KEY] + try { + return loadNativeParser() !== null + } finally { + if (saved !== undefined) process.env[ENV_KEY] = saved + } +} + +async function parseBothWays(filename: string, content: string, nativeLogs?: string[]) { + const dir = mkdtempSync(join(tmpdir(), 'chatlab-json-parity-')) + const filePath = join(dir, filename) + try { + writeFileSync(filePath, content, 'utf-8') + assert.equal(detectFormat(filePath)?.id, 'chatlab') + + delete process.env[ENV_KEY] + const nativeResult = nativeLogs ? await parseCollectingLogs(filePath, nativeLogs) : await parseFileSync(filePath) + + process.env[ENV_KEY] = '1' + const tsResult = await parseFileSync(filePath) + + return { nativeResult, tsResult } + } finally { + delete process.env[ENV_KEY] + rmSync(dir, { recursive: true, force: true }) + } +} + +function assertParity(nativeResult: ParseResult, tsResult: ParseResult) { + assert.deepEqual(nativeResult.meta, tsResult.meta) + assert.deepEqual(nativeResult.members, tsResult.members) + assert.equal(nativeResult.messages.length, tsResult.messages.length) + for (let i = 0; i < tsResult.messages.length; i++) { + assert.deepEqual(nativeResult.messages[i], tsResult.messages[i], `message #${i} differs`) + } +} + +// Canonical fixture without member roles (the TS head-regex parses this +// members array successfully, so both implementations use the head members). +const CANONICAL_FIXTURE = { + chatlab: { version: '0.0.2', exportedAt: 1700000000, generator: 'ChatLab' }, + meta: { + name: '测试群', + platform: 'weixin', + type: 'group', + groupId: 'g1@chatroom', + groupAvatar: 'https://example.com/g.jpg', + }, + members: [ + { platformId: 'u1', accountName: 'Alice', groupNickname: '小A', avatar: 'https://example.com/a.jpg' }, + { platformId: 'u2', accountName: 'Bob' }, + ], + messages: [ + { + platformMessageId: 'm1', + sender: 'u1', + accountName: 'Alice', + groupNickname: '小A', + timestamp: 1700000100, + type: 0, + content: '你好', + }, + { + platformMessageId: 'm2', + sender: 'u2', + accountName: 'Bob', + timestamp: 1700000200, + type: 25, + content: '回复内容', + replyToMessageId: 'm1', + }, + // null content (e.g. image message) and no optional ids + { sender: 'u1', accountName: 'Alice', groupNickname: '小A', timestamp: 1700000300, type: 1, content: null }, + // empty-string content passes through as-is + { sender: 'u2', accountName: 'Bob', timestamp: 1700000400, type: 0, content: '' }, + ], +} + +describe('chatlab native parser parity', { skip: !nativeAvailable() && 'native module not built' }, () => { + it('produces identical output to the TS parser on a canonical file', async () => { + const { nativeResult, tsResult } = await parseBothWays('测试群.json', JSON.stringify(CANONICAL_FIXTURE, null, 2)) + assertParity(nativeResult, tsResult) + + assert.equal(nativeResult.meta.name, '测试群') + assert.equal(nativeResult.meta.platform, 'weixin') + assert.equal(nativeResult.meta.groupId, 'g1@chatroom') + assert.equal(nativeResult.members.length, 2) + assert.equal(nativeResult.members[0].avatar, 'https://example.com/a.jpg') + assert.equal(nativeResult.messages.length, 4) + assert.equal(nativeResult.messages[1].replyToMessageId, 'm1') + assert.equal(nativeResult.messages[2].content, null) + assert.equal(nativeResult.messages[3].content, '') + }) + + it('collects members from messages when the members array is absent', async () => { + const noMembers = { + chatlab: { version: '0.0.2' }, + meta: { name: '无成员表', platform: 'qq', type: 'group' }, + messages: [ + { sender: 'u1', accountName: '旧名', groupNickname: '昵称', timestamp: 1, type: 0, content: 'a' }, + // Later message replaces the whole member entry (JS Map.set semantics) + { sender: 'u1', accountName: '新名', timestamp: 2, type: 0, content: 'b' }, + { sender: 'u2', accountName: 'B', timestamp: 3, type: 0, content: 'c' }, + ], + } + const { nativeResult, tsResult } = await parseBothWays('无成员表.json', JSON.stringify(noMembers)) + assertParity(nativeResult, tsResult) + assert.equal(nativeResult.members.length, 2) + assert.equal(nativeResult.members[0].accountName, '新名') + assert.equal(nativeResult.members[0].groupNickname, undefined) + }) + + it('handles missing meta name and empty messages identically', async () => { + const minimal = { + chatlab: { version: '0.0.2' }, + meta: { platform: 'weixin', type: 'private' }, + messages: [], + } + const { nativeResult, tsResult } = await parseBothWays('私聊记录.json', JSON.stringify(minimal)) + assertParity(nativeResult, tsResult) + // Default name falls back to the filename. + assert.equal(nativeResult.meta.name, '私聊记录') + assert.equal(nativeResult.meta.type, 'private') + assert.deepEqual(nativeResult.messages, []) + }) + + it('keeps member roles and avatars where the TS head-regex heuristic loses them', async () => { + // The TS parser extracts head members with a non-greedy regex that breaks + // on nested arrays (roles), silently degrading to message-collected + // members. The Rust kernel parses the members array structurally, so this + // is a deliberate improvement over TS behavior, not a parity case. + const withRoles = { + chatlab: { version: '0.0.2' }, + meta: { name: '带角色群', platform: 'qq', type: 'group' }, + members: [ + { + platformId: 'u1', + accountName: 'Alice', + avatar: 'https://example.com/a.jpg', + roles: [{ id: 'owner' }, { id: 'admin', name: '管理员' }], + }, + { platformId: 'u2', accountName: 'Bob' }, + ], + messages: [{ sender: 'u1', accountName: 'Alice', timestamp: 1, type: 0, content: 'x' }], + } + const { nativeResult, tsResult } = await parseBothWays('带角色群.json', JSON.stringify(withRoles)) + + // TS loses the head members entirely (only u1 appears, from messages). + assert.equal(tsResult.members.length, 1) + assert.equal(tsResult.members[0].avatar, undefined) + + // Native keeps the full member list including roles and avatars. + assert.equal(nativeResult.members.length, 2) + assert.equal(nativeResult.members[0].avatar, 'https://example.com/a.jpg') + assert.deepEqual(nativeResult.members[0].roles, [{ id: 'owner' }, { id: 'admin', name: '管理员' }]) + + // Meta and messages still match exactly. + assert.deepEqual(nativeResult.meta, tsResult.meta) + assert.deepEqual(nativeResult.messages, tsResult.messages) + }) + + it('falls back to the TS parser for off-spec files', async () => { + const offSpec = { + chatlab: { version: '0.0.2' }, + meta: { name: '异常文件', platform: 'qq', type: 'group' }, + messages: [ + // content must be string|null per spec; a number forces the fallback. + { sender: 'u1', accountName: 'A', timestamp: 1, type: 0, content: 42 }, + ], + } + const logs: string[] = [] + const { nativeResult, tsResult } = await parseBothWays('异常文件.json', JSON.stringify(offSpec), logs) + assert.ok( + logs.some((m) => m.includes('[NativeParser]') && m.includes('falling back')), + 'expected a fallback log entry' + ) + // Fallback means the native path output IS the TS output, quirks included. + assertParity(nativeResult, tsResult) + assert.equal(nativeResult.messages[0].content, 42 as unknown as string) + }) +}) diff --git a/packages/parser/src/native/chatlab-native.ts b/packages/parser/src/native/chatlab-native.ts new file mode 100644 index 00000000..af785e11 --- /dev/null +++ b/packages/parser/src/native/chatlab-native.ts @@ -0,0 +1,153 @@ +/** + * Rust-accelerated ChatLab JSON parsing with transparent TS fallback. + * + * Event order and payload shapes are identical to the pure-TS parser in + * formats/chatlab.ts for spec-compliant files (verified by parity tests). + * The Rust kernel is strict about the format spec: off-spec files (wrong + * field types, missing required fields) make parse() fail, and we fall back + * to the TS parser which replicates all JS passthrough quirks. Fallback also + * happens when the native module is unavailable or disabled via + * CHATLAB_DISABLE_NATIVE_PERF=1. + */ + +import type { ChatType, MessageType } from '@openchatlab/shared-types' +import type { ChatlabParser, NativeChatlabMember, NativeChatlabMessage } from '@openchatlab/parser-native' +import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import { getFileSize, createProgress } from '../utils' +import { loadNativeParser } from './loader' + +const PROGRESS_POLL_INTERVAL_MS = 200 + +type ParseGenerator = (options: ParseOptions) => AsyncGenerator + +function toParsedMessage(message: NativeChatlabMessage): ParsedMessage { + return { + senderPlatformId: message.senderPlatformId, + senderAccountName: message.senderAccountName, + senderGroupNickname: message.senderGroupNickname, + timestamp: message.timestamp, + type: message.messageType as MessageType, + // The kernel rejects a missing content key, so undefined here always + // means the source value was JSON null. + content: message.content ?? null, + platformMessageId: message.platformMessageId, + replyToMessageId: message.replyToMessageId, + } +} + +function toParsedMember(member: NativeChatlabMember, fromHead: boolean): ParsedMember { + if (!fromHead) { + // Members collected from messages carry only these three keys in the TS + // parser; keep the object shape identical. + return { + platformId: member.platformId, + accountName: member.accountName, + groupNickname: member.groupNickname, + } + } + return { + platformId: member.platformId, + accountName: member.accountName, + groupNickname: member.groupNickname, + avatar: member.avatar, + // Role objects pass through with their original key set ({id} or {id, name}). + roles: member.roles?.map((role) => (role.name !== undefined ? { id: role.id, name: role.name } : { id: role.id })), + } +} + +async function* parseChatLabNative( + options: ParseOptions, + parser: ChatlabParser, + fallback: ParseGenerator +): AsyncGenerator { + const { filePath, batchSize = 5000, onProgress, onLog } = options + + const totalBytes = getFileSize(filePath) + const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') + yield { type: 'progress', data: initialProgress } + onProgress?.(initialProgress) + onLog?.( + 'info', + `[NativeParser] Parsing ChatLab JSON with Rust kernel, size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB` + ) + + const progressTimer = setInterval(() => { + const progress = parser.progress() + onProgress?.( + createProgress( + 'parsing', + progress.bytesRead, + totalBytes, + progress.messagesProcessed, + `已处理 ${progress.messagesProcessed} 条消息...` + ) + ) + }, PROGRESS_POLL_INTERVAL_MS) + + try { + await parser.parse() + } catch (error) { + onLog?.( + 'warn', + `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + // Only progress events have been emitted so far — safe to restart with TS. + yield* fallback(options) + return + } finally { + clearInterval(progressTimer) + } + + const nativeMeta = parser.getMeta() + const meta: ParsedMeta = { + name: nativeMeta.name, + platform: nativeMeta.platform, + type: nativeMeta.chatType as ChatType, + groupId: nativeMeta.groupId, + groupAvatar: nativeMeta.groupAvatar, + } + yield { type: 'meta', data: meta } + + const members = parser.takeMembers().map((member) => toParsedMember(member, nativeMeta.membersFromHead)) + yield { type: 'members', data: members } + + let messagesProcessed = 0 + while (true) { + const batch = parser.takeBatch(batchSize) + if (!batch) break + messagesProcessed += batch.length + yield { type: 'messages', data: batch.map(toParsedMessage) } + } + + const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') + yield { type: 'progress', data: doneProgress } + onProgress?.(doneProgress) + onLog?.('info', `解析完成: ${messagesProcessed} 条消息, ${members.length} 个成员`) + + yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: members.length } } +} + +/** Wrap the TS ChatLab parse generator with native acceleration. */ +export function withNativeChatlab(fallback: ParseGenerator): ParseGenerator { + return async function* (options: ParseOptions): AsyncGenerator { + const native = loadNativeParser() + if (!native) { + yield* fallback(options) + return + } + + let parser: ChatlabParser + try { + parser = new native.ChatlabParser(options.filePath) + } catch (error) { + options.onLog?.( + 'warn', + `[NativeParser] Failed to create Rust parser, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + yield* fallback(options) + return + } + + yield* parseChatLabNative(options, parser, fallback) + } +} diff --git a/packages/parser/src/native/loader.ts b/packages/parser/src/native/loader.ts new file mode 100644 index 00000000..623c08ab --- /dev/null +++ b/packages/parser/src/native/loader.ts @@ -0,0 +1,44 @@ +/** + * Loader for the optional Rust parsing kernels (@openchatlab/parser-native). + * + * The native module is a local-build artifact: contributors without a Rust + * toolchain, published CLI installs and unsupported platforms simply fall + * back to the pure-TS parsers. Loading uses a runtime require so bundlers + * (electron-vite / tsup) never try to inline the .node binary. + */ + +import { createRequire } from 'node:module' + +import type { ChatlabParser, WeflowParser } from '@openchatlab/parser-native' + +export interface NativeParserModule { + WeflowParser: typeof WeflowParser + ChatlabParser: typeof ChatlabParser +} + +let cachedModule: NativeParserModule | null | undefined + +function isNativeDisabled(): boolean { + return process.env.CHATLAB_DISABLE_NATIVE_PERF === '1' +} + +function requireNativeModule(): NativeParserModule { + const requireFn = createRequire(import.meta.url) + return requireFn('@openchatlab/parser-native') as NativeParserModule +} + +/** + * Returns the native module, or null when disabled/unavailable. + * The module itself is cached; the env switch is evaluated on every call so + * tests can toggle it at runtime. + */ +export function loadNativeParser(): NativeParserModule | null { + if (isNativeDisabled()) return null + if (cachedModule !== undefined) return cachedModule + try { + cachedModule = requireNativeModule() + } catch { + cachedModule = null + } + return cachedModule +} diff --git a/packages/parser/src/native/weflow-native.test.ts b/packages/parser/src/native/weflow-native.test.ts new file mode 100644 index 00000000..f96631b4 --- /dev/null +++ b/packages/parser/src/native/weflow-native.test.ts @@ -0,0 +1,264 @@ +/** + * Parity tests: the Rust WeFlow kernel must produce byte-for-byte identical + * ParseResult output to the pure-TS stream-json parser. + * + * Skipped automatically when the native module has not been built locally + * (pnpm build:native). + */ + +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, it } from 'node:test' + +import { detectFormat, parseFileSync } from '../index' +import type { ParseResult } from '../types' +import { loadNativeParser } from './loader' + +const ENV_KEY = 'CHATLAB_DISABLE_NATIVE_PERF' + +function nativeAvailable(): boolean { + const saved = process.env[ENV_KEY] + delete process.env[ENV_KEY] + try { + return loadNativeParser() !== null + } finally { + if (saved !== undefined) process.env[ENV_KEY] = saved + } +} + +async function parseBothWays(filename: string, content: string, expectedFormatId: string) { + const dir = mkdtempSync(join(tmpdir(), 'chatlab-weflow-parity-')) + const filePath = join(dir, filename) + try { + writeFileSync(filePath, content, 'utf-8') + assert.equal(detectFormat(filePath)?.id, expectedFormatId) + + delete process.env[ENV_KEY] + const nativeResult = await parseFileSync(filePath) + + process.env[ENV_KEY] = '1' + const tsResult = await parseFileSync(filePath) + + return { nativeResult, tsResult } + } finally { + delete process.env[ENV_KEY] + rmSync(dir, { recursive: true, force: true }) + } +} + +function assertParity(nativeResult: ParseResult, tsResult: ParseResult) { + assert.deepEqual(nativeResult.meta, tsResult.meta) + assert.deepEqual(nativeResult.members, tsResult.members) + assert.equal(nativeResult.messages.length, tsResult.messages.length) + for (let i = 0; i < tsResult.messages.length; i++) { + assert.deepEqual(nativeResult.messages[i], tsResult.messages[i], `message #${i} differs`) + } +} + +// Fixture covering every branch of processMessage / meta extraction: +// avatars lookup, @chatroom skip, missing/empty sender, missing/null +// createTime, non-string content, whitespace-only content, renamed member, +// senderAvatarKey fallback, unknown message type, ownerId inference. +const FULL_FIXTURE = { + weflow: { version: '1.2.3', exportedAt: 1704164645 }, + session: { + wxid: 'room123@chatroom', + nickname: '昵称群', + remark: '', + displayName: '显示名群', + type: '群聊', + lastTimestamp: 1704200000, + messageCount: 10, + avatar: '', + }, + avatars: { + wxid_alice: 'data:image/jpeg;base64,ALICE', + 'room123@chatroom': 'data:image/jpeg;base64,ROOM', + wxid_keyed: 'data:image/jpeg;base64,KEYED', + wxid_empty: '', + }, + messages: [ + { + localId: 1, + createTime: 1704164645, + formattedTime: '', + type: '文本消息', + localType: 1, + content: ' hello world ', + isSend: 0, + senderUsername: 'wxid_alice', + senderDisplayName: 'Alice', + senderAvatarKey: 'wxid_alice', + source: '', + }, + // Owner message (isSend=1) + { + localId: 2, + createTime: 1704164700, + type: '图片消息', + content: '[图片]', + isSend: 1, + senderUsername: 'wxid_owner', + senderDisplayName: 'Me', + senderAvatarKey: '', + source: '', + }, + // System message from the room itself -> skipped + { + localId: 3, + createTime: 1704164710, + type: '系统消息', + content: 'joined', + isSend: null, + senderUsername: 'room123@chatroom', + senderDisplayName: '群', + }, + // Missing senderUsername -> skipped + { localId: 4, createTime: 1704164720, type: '文本消息', content: 'no sender' }, + // Empty senderUsername -> skipped + { localId: 5, createTime: 1704164725, type: '文本消息', content: 'x', senderUsername: '' }, + // Missing createTime -> skipped + { localId: 6, type: '文本消息', content: 'no time', senderUsername: 'wxid_alice' }, + // Null createTime -> kept with null timestamp (importer will skip) + { + localId: 7, + createTime: null, + type: '文本消息', + content: 'null time', + senderUsername: 'wxid_alice', + senderDisplayName: 'Alice', + }, + // Non-string content -> JSON.stringify + { + localId: 8, + createTime: 1704164730, + type: '引用消息', + content: { title: '引用', items: [1, true, null], nested: { a: 'b' } }, + isSend: 0, + senderUsername: 'wxid_alice', + senderDisplayName: 'Alice Renamed', + }, + // Whitespace-only content -> null; avatar via senderAvatarKey pointing elsewhere + { + localId: 9, + createTime: 1704164740, + type: '未知类型(42)', + content: ' ', + isSend: 0, + senderUsername: 'wxid_bob', + senderDisplayName: '', + senderAvatarKey: 'wxid_keyed', + }, + // Numeric localId with fraction, emoji content, unicode whitespace trim + { + localId: 10.5, + createTime: 1704164750, + type: '动画表情', + content: '\u00a0😀 表情 \u3000', + isSend: 0, + senderUsername: 'wxid_owner', + senderDisplayName: 'Me2', + }, + // Missing localId -> String(undefined); missing content -> null + { + createTime: 1704164760, + type: '红包卡片', + isSend: 0, + senderUsername: 'wxid_alice', + senderDisplayName: 'Alice Renamed', + }, + ], +} + +describe('weflow native parser parity', { skip: !nativeAvailable() && 'native module not built' }, () => { + it('produces identical output to the TS parser on the full fixture', async () => { + const { nativeResult, tsResult } = await parseBothWays( + '显示名群.json', + JSON.stringify(FULL_FIXTURE, null, 2), + 'weflow' + ) + assertParity(nativeResult, tsResult) + + // Sanity-check key semantics directly (guards against both parsers + // being wrong in the same way on the highest-risk branches). + assert.equal(nativeResult.meta.name, '显示名群') + assert.equal(nativeResult.meta.groupId, 'room123@chatroom') + assert.equal(nativeResult.meta.groupAvatar, 'data:image/jpeg;base64,ROOM') + assert.equal(nativeResult.meta.ownerId, 'wxid_owner') + assert.equal(nativeResult.messages.length, 7) + assert.equal(nativeResult.messages[0].content, 'hello world') + assert.equal(nativeResult.messages[2].timestamp, null) + assert.equal(nativeResult.messages[3].content, '{"title":"引用","items":[1,true,null],"nested":{"a":"b"}}') + assert.equal(nativeResult.messages[4].content, null) + assert.equal(nativeResult.messages[5].platformMessageId, '10.5') + assert.equal(nativeResult.messages[6].platformMessageId, 'undefined') + const bob = nativeResult.members.find((m) => m.platformId === 'wxid_bob') + assert.equal(bob?.avatar, 'data:image/jpeg;base64,KEYED') + assert.equal(bob?.accountName, 'wxid_bob') + const alice = nativeResult.members.find((m) => m.platformId === 'wxid_alice') + assert.equal(alice?.accountName, 'Alice Renamed') + }) + + it('handles private chats and missing session identically', async () => { + const privateChat = { + weflow: { version: '1.0.0' }, + session: { wxid: 'wxid_friend', type: '私聊', nickname: '好友', displayName: '' }, + avatars: {}, + messages: [ + { + localId: 1, + createTime: 100, + type: '文本消息', + content: 'hi', + isSend: 1, + senderUsername: 'wxid_me', + senderDisplayName: '我', + }, + ], + } + const { nativeResult, tsResult } = await parseBothWays('好友聊天.json', JSON.stringify(privateChat), 'weflow') + assertParity(nativeResult, tsResult) + assert.equal(nativeResult.meta.type, 'private') + assert.equal(nativeResult.meta.groupId, undefined) + assert.equal(nativeResult.meta.name, '好友') + }) + + it('handles the echotrace variant (shared parser) identically', async () => { + const echotrace = { + session: { wxid: 'room9@chatroom', type: '群聊', displayName: 'Echo 群' }, + messages: [ + { + localId: 1, + createTime: 200, + type: '文本消息', + content: 'echo', + isSend: 0, + senderUsername: 'wxid_e', + senderDisplayName: 'E', + }, + ], + } + const { nativeResult, tsResult } = await parseBothWays( + 'echo.json', + JSON.stringify(echotrace), + 'ycccccccy-echotrace' + ) + assertParity(nativeResult, tsResult) + assert.equal(nativeResult.messages.length, 1) + }) + + it('handles empty message arrays identically', async () => { + const empty = { + weflow: { version: '1.0.0' }, + session: { wxid: 'r@chatroom', type: '群聊', displayName: '空群' }, + avatars: {}, + messages: [], + } + const { nativeResult, tsResult } = await parseBothWays('空群.json', JSON.stringify(empty), 'weflow') + assert.deepEqual(nativeResult.meta, tsResult.meta) + assert.deepEqual(nativeResult.members, tsResult.members) + assert.deepEqual(nativeResult.messages, tsResult.messages) + }) +}) diff --git a/packages/parser/src/native/weflow-native.ts b/packages/parser/src/native/weflow-native.ts new file mode 100644 index 00000000..a383aa44 --- /dev/null +++ b/packages/parser/src/native/weflow-native.ts @@ -0,0 +1,138 @@ +/** + * Rust-accelerated WeFlow parsing with transparent TS fallback. + * + * Event order and payload shapes are identical to the pure-TS parser in + * formats/weflow.ts (verified by parity tests): progress → meta → members → + * message batches → done. Fallback happens when the native module is + * unavailable, disabled via CHATLAB_DISABLE_NATIVE_PERF=1, or fails before + * any data event has been emitted. + */ + +import { KNOWN_PLATFORMS, ChatType, type MessageType } from '@openchatlab/shared-types' +import type { NativeParsedMessage, WeflowParser } from '@openchatlab/parser-native' +import type { ParseEvent, ParseOptions, ParsedMessage, ParsedMeta } from '../types' +import { getFileSize, createProgress } from '../utils' +import { loadNativeParser } from './loader' + +const PROGRESS_POLL_INTERVAL_MS = 200 + +type ParseGenerator = (options: ParseOptions) => AsyncGenerator + +function toParsedMessage(message: NativeParsedMessage): ParsedMessage { + return { + platformMessageId: message.platformMessageId, + senderPlatformId: message.senderPlatformId, + senderAccountName: message.senderAccountName, + // WeFlow has no separate group nickname field (same as the TS parser). + senderGroupNickname: undefined, + // The TS parser passes createTime through as-is, including null. + timestamp: (message.timestamp ?? null) as unknown as number, + type: message.messageType as MessageType, + content: message.content ?? null, + } +} + +async function* parseWeFlowNative( + options: ParseOptions, + parser: WeflowParser, + fallback: ParseGenerator +): AsyncGenerator { + const { filePath, batchSize = 5000, onProgress, onLog } = options + + const totalBytes = getFileSize(filePath) + const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') + yield { type: 'progress', data: initialProgress } + onProgress?.(initialProgress) + onLog?.( + 'info', + `[NativeParser] Parsing WeFlow export with Rust kernel, size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB` + ) + + const progressTimer = setInterval(() => { + const progress = parser.progress() + onProgress?.( + createProgress( + 'parsing', + progress.bytesRead, + totalBytes, + progress.messagesProcessed, + `已处理 ${progress.messagesProcessed} 条消息...` + ) + ) + }, PROGRESS_POLL_INTERVAL_MS) + + try { + await parser.parse() + } catch (error) { + onLog?.( + 'warn', + `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + // Only progress events have been emitted so far — safe to restart with TS. + yield* fallback(options) + return + } finally { + clearInterval(progressTimer) + } + + const nativeMeta = parser.getMeta() + const meta: ParsedMeta = { + name: nativeMeta.name, + platform: KNOWN_PLATFORMS.WECHAT, + type: nativeMeta.chatType === 'private' ? ChatType.PRIVATE : ChatType.GROUP, + groupId: nativeMeta.groupId, + groupAvatar: nativeMeta.groupAvatar, + ownerId: nativeMeta.ownerId, + } + yield { type: 'meta', data: meta } + + const members = parser.takeMembers().map((member) => ({ + platformId: member.platformId, + accountName: member.accountName, + avatar: member.avatar, + })) + yield { type: 'members', data: members } + + let messagesProcessed = 0 + while (true) { + const batch = parser.takeBatch(batchSize) + if (!batch) break + messagesProcessed += batch.length + yield { type: 'messages', data: batch.map(toParsedMessage) } + } + + const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') + yield { type: 'progress', data: doneProgress } + onProgress?.(doneProgress) + onLog?.('info', `解析完成: ${messagesProcessed} 条消息, ${members.length} 个成员`) + + yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: members.length } } +} + +/** + * Wrap a TS parse generator with native acceleration. Used by the WeFlow and + * ycccccccy-echotrace format modules (they share the same data structure). + */ +export function withNativeWeflow(fallback: ParseGenerator): ParseGenerator { + return async function* (options: ParseOptions): AsyncGenerator { + const native = loadNativeParser() + if (!native) { + yield* fallback(options) + return + } + + let parser: WeflowParser + try { + parser = new native.WeflowParser(options.filePath) + } catch (error) { + options.onLog?.( + 'warn', + `[NativeParser] Failed to create Rust parser, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + yield* fallback(options) + return + } + + yield* parseWeFlowNative(options, parser, fallback) + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6bc83796..5695ba2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -197,6 +197,9 @@ importers: '@node-rs/jieba': specifier: ^2.0.1 version: 2.0.1 + '@openchatlab/parser-native': + specifier: workspace:* + version: link:../../packages/parser-native better-sqlite3: specifier: ^12.4.6 version: 12.6.2 @@ -378,6 +381,9 @@ importers: packages/parser: dependencies: + '@openchatlab/parser-native': + specifier: workspace:* + version: link:../parser-native '@openchatlab/shared-types': specifier: workspace:* version: link:../shared-types @@ -389,6 +395,12 @@ importers: specifier: ^1.7.8 version: 1.7.8 + packages/parser-native: + devDependencies: + '@napi-rs/cli': + specifier: ^3.0.0 + version: 3.7.2(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)(@types/node@25.2.2) + packages/shared-types: {} packages/sync: {} @@ -1516,6 +1528,140 @@ packages: cpu: [x64] os: [win32] + '@inquirer/ansi@2.0.7': + resolution: {integrity: sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + + '@inquirer/checkbox@5.2.1': + resolution: {integrity: sha512-b6xmA/VlTe0ZgDQHDui+Nav470u7u49nRd8/iuhOcQPO9Ch7lGuogydhi2VOmNlZ+zXcM8IcPuNSwQcdJaF/kw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/confirm@6.1.1': + resolution: {integrity: sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/core@11.2.1': + resolution: {integrity: sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/editor@5.2.2': + resolution: {integrity: sha512-ZRVd/oD+sYsUd5zVm0NflqEzlqfYCyHNsqkHl2oWXEUHs12tCbcSFi+wVFEvD8+LGRaMUsVrE7qeo6lSG/S1Vg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/expand@5.1.1': + resolution: {integrity: sha512-YmQpenjbFSHAK3sOd44puHh3V1KXXr+JiNpUztoSQ4drLh2rTVzTap/YtlAVu/5xavifIlBfNEzJ/neZJ1a/1g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/external-editor@3.0.3': + resolution: {integrity: sha512-6thf5I8q7lZwzGLAxPaaGEREEkZ3nyePPDQ1oyobblxmEE8mqTLguScP7pDjUTAibiyb4hfXl+qjUEJ+di/aNA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/figures@2.0.7': + resolution: {integrity: sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + + '@inquirer/input@5.1.2': + resolution: {integrity: sha512-9K/DDBSQpOyZSkt6sOVP9Vo0TR7atX2kuILsUu0x3wVcVbe97lJwIJKMLdMw25tDYuXl/qp6erT0Xs1rfmcfZg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/number@4.1.1': + resolution: {integrity: sha512-XF4IXAbPnGPgw0wsbC/i2tPcyfdZgDpUlhsqU0SfT4IRIGWha6Xm9VRgN5yYxJq+jnyXlfXI/nQ3ulfk0iEICA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/password@5.1.1': + resolution: {integrity: sha512-3XBfF7DAsp5qeDsvN5Rd1HmbNokVvEQoUM0QLrRcybC9nX96w3Pbmu7qUsb3IT3J3jBvs2+mTXaKHOUsgHMLzg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/prompts@8.5.2': + resolution: {integrity: sha512-IYR/3C/paEVVQYQvdDlFZVjRCJVYHHON0XXMH91KO9GSxs0TdKYWlUdvfQl2EfAHDxUaN3IBffkE/BDTh5nJ6g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/rawlist@5.3.1': + resolution: {integrity: sha512-QqdTqQddL3qPX/PPrjobpsO25NZ4dWXgTLenrR445L2ptLEYE6Z+PD5c5CNDJNx4ugRgELAIpSIJxZaO2jJ2Og==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/search@4.2.1': + resolution: {integrity: sha512-xJj8QWKRSrfKoBIITLZK61dD3zwo0Rz11fgDImku30/Oe81zMdIdGgrLY2h6RkJ+KZ/GhNYIRMKnH/62qBTA5g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/select@5.2.1': + resolution: {integrity: sha512-FlDndEUww8m7BfukO2nJa25vhD+H5jxxCv4oGioKqzyWz3nPHhhw4LKdYRSlXuAx7DsdWia7iyaBPKKS95Evfw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/type@4.0.7': + resolution: {integrity: sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + '@internationalized/date@3.11.0': resolution: {integrity: sha512-BOx5huLAWhicM9/ZFs84CzP+V3gBW6vlpM02yzsdYC7TGlZJX1OJiEEHcSayF00Z+3jLlm4w79amvSt6RqKN3Q==} @@ -1583,9 +1729,345 @@ packages: '@cfworker/json-schema': optional: true + '@napi-rs/cli@3.7.2': + resolution: {integrity: sha512-shDW0Td/XZQpP04Yy+OsMt1ILMKGGkoLcy1zVAsSAK0fLfWm0Upgkmfs/NOV2ZhMQwkgpR3ZEdyHmTwgrUDQuA==} + engines: {node: '>= 16'} + hasBin: true + peerDependencies: + '@emnapi/runtime': ^1.7.1 + peerDependenciesMeta: + '@emnapi/runtime': + optional: true + + '@napi-rs/cross-toolchain@1.0.3': + resolution: {integrity: sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg==} + peerDependencies: + '@napi-rs/cross-toolchain-arm64-target-aarch64': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-armv7': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-ppc64le': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-s390x': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-x86_64': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-aarch64': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-armv7': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-ppc64le': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-s390x': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-x86_64': ^1.0.3 + peerDependenciesMeta: + '@napi-rs/cross-toolchain-arm64-target-aarch64': + optional: true + '@napi-rs/cross-toolchain-arm64-target-armv7': + optional: true + '@napi-rs/cross-toolchain-arm64-target-ppc64le': + optional: true + '@napi-rs/cross-toolchain-arm64-target-s390x': + optional: true + '@napi-rs/cross-toolchain-arm64-target-x86_64': + optional: true + '@napi-rs/cross-toolchain-x64-target-aarch64': + optional: true + '@napi-rs/cross-toolchain-x64-target-armv7': + optional: true + '@napi-rs/cross-toolchain-x64-target-ppc64le': + optional: true + '@napi-rs/cross-toolchain-x64-target-s390x': + optional: true + '@napi-rs/cross-toolchain-x64-target-x86_64': + optional: true + + '@napi-rs/lzma-android-arm-eabi@1.4.5': + resolution: {integrity: sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/lzma-android-arm64@1.4.5': + resolution: {integrity: sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/lzma-darwin-arm64@1.4.5': + resolution: {integrity: sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/lzma-darwin-x64@1.4.5': + resolution: {integrity: sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/lzma-freebsd-x64@1.4.5': + resolution: {integrity: sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/lzma-linux-arm-gnueabihf@1.4.5': + resolution: {integrity: sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/lzma-linux-arm64-gnu@1.4.5': + resolution: {integrity: sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/lzma-linux-arm64-musl@1.4.5': + resolution: {integrity: sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/lzma-linux-ppc64-gnu@1.4.5': + resolution: {integrity: sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + + '@napi-rs/lzma-linux-riscv64-gnu@1.4.5': + resolution: {integrity: sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/lzma-linux-s390x-gnu@1.4.5': + resolution: {integrity: sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + + '@napi-rs/lzma-linux-x64-gnu@1.4.5': + resolution: {integrity: sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/lzma-linux-x64-musl@1.4.5': + resolution: {integrity: sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/lzma-wasm32-wasi@1.4.5': + resolution: {integrity: sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/lzma-win32-arm64-msvc@1.4.5': + resolution: {integrity: sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/lzma-win32-ia32-msvc@1.4.5': + resolution: {integrity: sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/lzma-win32-x64-msvc@1.4.5': + resolution: {integrity: sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/lzma@1.4.5': + resolution: {integrity: sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg==} + engines: {node: '>= 10'} + + '@napi-rs/tar-android-arm-eabi@1.1.0': + resolution: {integrity: sha512-h2Ryndraj/YiKgMV/r5by1cDusluYIRT0CaE0/PekQ4u+Wpy2iUVqvzVU98ZPnhXaNeYxEvVJHNGafpOfaD0TA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/tar-android-arm64@1.1.0': + resolution: {integrity: sha512-DJFyQHr1ZxNZorm/gzc1qBNLF/FcKzcH0V0Vwan5P+o0aE2keQIGEjJ09FudkF9v6uOuJjHCVDdK6S6uHtShAw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/tar-darwin-arm64@1.1.0': + resolution: {integrity: sha512-Zz2sXRzjIX4e532zD6xm2SjXEym6MkvfCvL2RMpG2+UwNVDVscHNcz3d47Pf3sysP2e2af7fBB3TIoK2f6trPw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/tar-darwin-x64@1.1.0': + resolution: {integrity: sha512-EI+CptIMNweT0ms9S3mkP/q+J6FNZ1Q6pvpJOEcWglRfyfQpLqjlC0O+dptruTPE8VamKYuqdjxfqD8hifZDOA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/tar-freebsd-x64@1.1.0': + resolution: {integrity: sha512-J0PIqX+pl6lBIAckL/c87gpodLbjZB1OtIK+RDscKC9NLdpVv6VGOxzUV/fYev/hctcE8EfkLbgFOfpmVQPg2g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/tar-linux-arm-gnueabihf@1.1.0': + resolution: {integrity: sha512-SLgIQo3f3EjkZ82ZwvrEgFvMdDAhsxCYjyoSuWfHCz0U16qx3SuGCp8+FYOPYCECHN3ZlGjXnoAIt9ERd0dEUg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/tar-linux-arm64-gnu@1.1.0': + resolution: {integrity: sha512-d014cdle52EGaH6GpYTQOP9Py7glMO1zz/+ynJPjjzYFSxvdYx0byrjumZk2UQdIyGZiJO2MEFpCkEEKFSgPYA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/tar-linux-arm64-musl@1.1.0': + resolution: {integrity: sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/tar-linux-ppc64-gnu@1.1.0': + resolution: {integrity: sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + + '@napi-rs/tar-linux-s390x-gnu@1.1.0': + resolution: {integrity: sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + + '@napi-rs/tar-linux-x64-gnu@1.1.0': + resolution: {integrity: sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/tar-linux-x64-musl@1.1.0': + resolution: {integrity: sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/tar-wasm32-wasi@1.1.0': + resolution: {integrity: sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/tar-win32-arm64-msvc@1.1.0': + resolution: {integrity: sha512-vfpG71OB0ijtjemp3WTdmBKJm9R70KM8vsSExMsIQtV0lVzP07oM1CW6JbNRPXNLhRoue9ofYLiUDk8bE0Hckg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/tar-win32-ia32-msvc@1.1.0': + resolution: {integrity: sha512-hGPyPW60YSpOSgzfy68DLBHgi6HxkAM+L59ZZZPMQ0TOXjQg+p2EW87+TjZfJOkSpbYiEkULwa/f4a2hcVjsqQ==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/tar-win32-x64-msvc@1.1.0': + resolution: {integrity: sha512-L6Ed1DxXK9YSCMyvpR8MiNAyKNkQLjsHsHK9E0qnHa8NzLFqzDKhvs5LfnWxM2kJ+F7m/e5n9zPm24kHb3LsVw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/tar@1.1.0': + resolution: {integrity: sha512-7cmzIu+Vbupriudo7UudoMRH2OA3cTw67vva8MxeoAe5S7vPFI7z0vp0pMXiA25S8IUJefImQ90FeJjl8fjEaQ==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@napi-rs/wasm-runtime@1.1.6': + resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + + '@napi-rs/wasm-tools-android-arm-eabi@1.0.1': + resolution: {integrity: sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/wasm-tools-android-arm64@1.0.1': + resolution: {integrity: sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/wasm-tools-darwin-arm64@1.0.1': + resolution: {integrity: sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/wasm-tools-darwin-x64@1.0.1': + resolution: {integrity: sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/wasm-tools-freebsd-x64@1.0.1': + resolution: {integrity: sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1': + resolution: {integrity: sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/wasm-tools-linux-arm64-musl@1.0.1': + resolution: {integrity: sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/wasm-tools-linux-x64-gnu@1.0.1': + resolution: {integrity: sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/wasm-tools-linux-x64-musl@1.0.1': + resolution: {integrity: sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/wasm-tools-wasm32-wasi@1.0.1': + resolution: {integrity: sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1': + resolution: {integrity: sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1': + resolution: {integrity: sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/wasm-tools-win32-x64-msvc@1.0.1': + resolution: {integrity: sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/wasm-tools@1.0.1': + resolution: {integrity: sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ==} + engines: {node: '>= 10'} + '@nodable/entities@2.1.0': resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==} @@ -1747,6 +2229,58 @@ packages: '@nuxtjs/color-mode@3.5.2': resolution: {integrity: sha512-cC6RfgZh3guHBMLLjrBB2Uti5eUoGM9KyauOaYS9ETmxNWBMTvpgjvSiSJp1OFljIXPIqVTJ3xtJpSNZiO3ZaA==} + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.3': + resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.11': + resolution: {integrity: sha512-+s7HUxjfFqOMS9VlIwDffq0MikjSAK0gSpG73W+meAvVAvX4MBrHYTK5Bj3Uot55qFT4gzUtfzE4mGWY4Br8/Q==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -2330,6 +2864,9 @@ packages: '@tybys/wasm-util@0.10.1': resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} + '@tybys/wasm-util@0.10.3': + resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} + '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} @@ -2851,6 +3388,9 @@ packages: resolution: {integrity: sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==} hasBin: true + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + better-sqlite3@12.6.2: resolution: {integrity: sha512-8VYKM3MjCa9WcaSAI3hzwhmyHVlH8tiGFwf0RlTsZPWJ1I5MkzjiudCo4KC4DxOaL/53A5B1sI/IbldNFDbsKA==} engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x} @@ -2980,6 +3520,9 @@ packages: character-entities-legacy@3.0.0: resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + chardet@2.2.0: + resolution: {integrity: sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -3016,6 +3559,15 @@ packages: resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==} engines: {node: '>=8'} + cli-width@4.1.0: + resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} + engines: {node: '>= 12'} + + clipanion@4.0.0-rc.4: + resolution: {integrity: sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q==} + peerDependencies: + typanion: '*' + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -3034,6 +3586,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + colortranslator@5.0.0: resolution: {integrity: sha512-Z3UPUKasUVDFCDYAjP2fmlVRf1jFHJv1izAmPjiOa0OCIw1W7iC8PZ2GsoDa8uZv+mKyWopxxStT9q05+27h7w==} @@ -3085,6 +3640,10 @@ packages: resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} engines: {node: '>= 0.6'} + content-type@2.0.0: + resolution: {integrity: sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==} + engines: {node: '>=18'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -3346,6 +3905,14 @@ packages: embla-carousel@8.6.0: resolution: {integrity: sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==} + emnapi@1.11.1: + resolution: {integrity: sha512-kSRjhIcxjMFsBqk7ORvoc9aA5SBKDmecrtF5RMcmOTao0kD/zamaxsuTxMI8C1//wGUuvE7a+19pCE7AEhGVnA==} + peerDependencies: + node-addon-api: '>= 6.1.0' + peerDependenciesMeta: + node-addon-api: + optional: true + emoji-regex-xs@1.0.0: resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} @@ -3400,6 +3967,9 @@ packages: resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} engines: {node: '>= 0.4'} + es-toolkit@1.49.0: + resolution: {integrity: sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==} + es6-error@4.1.1: resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} @@ -3600,9 +4170,18 @@ packages: fast-querystring@1.1.2: resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + fast-string-truncated-width@3.0.3: + resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} + + fast-string-width@3.0.2: + resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} + fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + fast-wrap-ansi@0.2.2: + resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} + fast-xml-builder@1.2.0: resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} @@ -4117,6 +4696,10 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true + js-yaml@4.3.0: + resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} + hasBin: true + jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -4150,6 +4733,9 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json-with-bigint@3.5.8: + resolution: {integrity: sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -4566,6 +5152,10 @@ packages: muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} + mute-stream@3.0.0: + resolution: {integrity: sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==} + engines: {node: ^20.17.0 || >=22.9.0} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -4661,6 +5251,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + obug@2.1.3: + resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} + engines: {node: '>=12.20.0'} + ofetch@1.5.1: resolution: {integrity: sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==} @@ -5244,6 +5838,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + send@1.2.1: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} @@ -5629,6 +6228,9 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + typanion@3.14.0: + resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -5716,6 +6318,9 @@ packages: unist-util-visit@5.1.0: resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -7278,6 +7883,125 @@ snapshots: '@img/sharp-win32-x64@0.34.5': optional: true + '@inquirer/ansi@2.0.7': {} + + '@inquirer/checkbox@5.2.1(@types/node@25.2.2)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/confirm@6.1.1(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/core@11.2.1(@types/node@25.2.2)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@25.2.2) + cli-width: 4.1.0 + fast-wrap-ansi: 0.2.2 + mute-stream: 3.0.0 + signal-exit: 4.1.0 + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/editor@5.2.2(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/external-editor': 3.0.3(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/expand@5.1.1(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/external-editor@3.0.3(@types/node@25.2.2)': + dependencies: + chardet: 2.2.0 + iconv-lite: 0.7.2 + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/figures@2.0.7': {} + + '@inquirer/input@5.1.2(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/number@4.1.1(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/password@5.1.1(@types/node@25.2.2)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/prompts@8.5.2(@types/node@25.2.2)': + dependencies: + '@inquirer/checkbox': 5.2.1(@types/node@25.2.2) + '@inquirer/confirm': 6.1.1(@types/node@25.2.2) + '@inquirer/editor': 5.2.2(@types/node@25.2.2) + '@inquirer/expand': 5.1.1(@types/node@25.2.2) + '@inquirer/input': 5.1.2(@types/node@25.2.2) + '@inquirer/number': 4.1.1(@types/node@25.2.2) + '@inquirer/password': 5.1.1(@types/node@25.2.2) + '@inquirer/rawlist': 5.3.1(@types/node@25.2.2) + '@inquirer/search': 4.2.1(@types/node@25.2.2) + '@inquirer/select': 5.2.1(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/rawlist@5.3.1(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/search@4.2.1(@types/node@25.2.2)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/select@5.2.1(@types/node@25.2.2)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@25.2.2) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@25.2.2) + optionalDependencies: + '@types/node': 25.2.2 + + '@inquirer/type@4.0.7(@types/node@25.2.2)': + optionalDependencies: + '@types/node': 25.2.2 + '@internationalized/date@3.11.0': dependencies: '@swc/helpers': 0.5.18 @@ -7376,6 +8100,202 @@ snapshots: transitivePeerDependencies: - supports-color + '@napi-rs/cli@3.7.2(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)(@types/node@25.2.2)': + dependencies: + '@inquirer/prompts': 8.5.2(@types/node@25.2.2) + '@napi-rs/cross-toolchain': 1.0.3(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@napi-rs/wasm-tools': 1.0.1(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@octokit/rest': 22.0.1 + clipanion: 4.0.0-rc.4(typanion@3.14.0) + colorette: 2.0.20 + emnapi: 1.11.1 + es-toolkit: 1.49.0 + js-yaml: 4.3.0 + obug: 2.1.3 + semver: 7.8.5 + typanion: 3.14.0 + optionalDependencies: + '@emnapi/runtime': 1.8.1 + transitivePeerDependencies: + - '@emnapi/core' + - '@napi-rs/cross-toolchain-arm64-target-aarch64' + - '@napi-rs/cross-toolchain-arm64-target-armv7' + - '@napi-rs/cross-toolchain-arm64-target-ppc64le' + - '@napi-rs/cross-toolchain-arm64-target-s390x' + - '@napi-rs/cross-toolchain-arm64-target-x86_64' + - '@napi-rs/cross-toolchain-x64-target-aarch64' + - '@napi-rs/cross-toolchain-x64-target-armv7' + - '@napi-rs/cross-toolchain-x64-target-ppc64le' + - '@napi-rs/cross-toolchain-x64-target-s390x' + - '@napi-rs/cross-toolchain-x64-target-x86_64' + - '@types/node' + - node-addon-api + - supports-color + + '@napi-rs/cross-toolchain@1.0.3(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + dependencies: + '@napi-rs/lzma': 1.4.5(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@napi-rs/tar': 1.1.0(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + debug: 4.4.3 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + - supports-color + + '@napi-rs/lzma-android-arm-eabi@1.4.5': + optional: true + + '@napi-rs/lzma-android-arm64@1.4.5': + optional: true + + '@napi-rs/lzma-darwin-arm64@1.4.5': + optional: true + + '@napi-rs/lzma-darwin-x64@1.4.5': + optional: true + + '@napi-rs/lzma-freebsd-x64@1.4.5': + optional: true + + '@napi-rs/lzma-linux-arm-gnueabihf@1.4.5': + optional: true + + '@napi-rs/lzma-linux-arm64-gnu@1.4.5': + optional: true + + '@napi-rs/lzma-linux-arm64-musl@1.4.5': + optional: true + + '@napi-rs/lzma-linux-ppc64-gnu@1.4.5': + optional: true + + '@napi-rs/lzma-linux-riscv64-gnu@1.4.5': + optional: true + + '@napi-rs/lzma-linux-s390x-gnu@1.4.5': + optional: true + + '@napi-rs/lzma-linux-x64-gnu@1.4.5': + optional: true + + '@napi-rs/lzma-linux-x64-musl@1.4.5': + optional: true + + '@napi-rs/lzma-wasm32-wasi@1.4.5(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + dependencies: + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + optional: true + + '@napi-rs/lzma-win32-arm64-msvc@1.4.5': + optional: true + + '@napi-rs/lzma-win32-ia32-msvc@1.4.5': + optional: true + + '@napi-rs/lzma-win32-x64-msvc@1.4.5': + optional: true + + '@napi-rs/lzma@1.4.5(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + optionalDependencies: + '@napi-rs/lzma-android-arm-eabi': 1.4.5 + '@napi-rs/lzma-android-arm64': 1.4.5 + '@napi-rs/lzma-darwin-arm64': 1.4.5 + '@napi-rs/lzma-darwin-x64': 1.4.5 + '@napi-rs/lzma-freebsd-x64': 1.4.5 + '@napi-rs/lzma-linux-arm-gnueabihf': 1.4.5 + '@napi-rs/lzma-linux-arm64-gnu': 1.4.5 + '@napi-rs/lzma-linux-arm64-musl': 1.4.5 + '@napi-rs/lzma-linux-ppc64-gnu': 1.4.5 + '@napi-rs/lzma-linux-riscv64-gnu': 1.4.5 + '@napi-rs/lzma-linux-s390x-gnu': 1.4.5 + '@napi-rs/lzma-linux-x64-gnu': 1.4.5 + '@napi-rs/lzma-linux-x64-musl': 1.4.5 + '@napi-rs/lzma-wasm32-wasi': 1.4.5(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@napi-rs/lzma-win32-arm64-msvc': 1.4.5 + '@napi-rs/lzma-win32-ia32-msvc': 1.4.5 + '@napi-rs/lzma-win32-x64-msvc': 1.4.5 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + + '@napi-rs/tar-android-arm-eabi@1.1.0': + optional: true + + '@napi-rs/tar-android-arm64@1.1.0': + optional: true + + '@napi-rs/tar-darwin-arm64@1.1.0': + optional: true + + '@napi-rs/tar-darwin-x64@1.1.0': + optional: true + + '@napi-rs/tar-freebsd-x64@1.1.0': + optional: true + + '@napi-rs/tar-linux-arm-gnueabihf@1.1.0': + optional: true + + '@napi-rs/tar-linux-arm64-gnu@1.1.0': + optional: true + + '@napi-rs/tar-linux-arm64-musl@1.1.0': + optional: true + + '@napi-rs/tar-linux-ppc64-gnu@1.1.0': + optional: true + + '@napi-rs/tar-linux-s390x-gnu@1.1.0': + optional: true + + '@napi-rs/tar-linux-x64-gnu@1.1.0': + optional: true + + '@napi-rs/tar-linux-x64-musl@1.1.0': + optional: true + + '@napi-rs/tar-wasm32-wasi@1.1.0(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + dependencies: + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + optional: true + + '@napi-rs/tar-win32-arm64-msvc@1.1.0': + optional: true + + '@napi-rs/tar-win32-ia32-msvc@1.1.0': + optional: true + + '@napi-rs/tar-win32-x64-msvc@1.1.0': + optional: true + + '@napi-rs/tar@1.1.0(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + optionalDependencies: + '@napi-rs/tar-android-arm-eabi': 1.1.0 + '@napi-rs/tar-android-arm64': 1.1.0 + '@napi-rs/tar-darwin-arm64': 1.1.0 + '@napi-rs/tar-darwin-x64': 1.1.0 + '@napi-rs/tar-freebsd-x64': 1.1.0 + '@napi-rs/tar-linux-arm-gnueabihf': 1.1.0 + '@napi-rs/tar-linux-arm64-gnu': 1.1.0 + '@napi-rs/tar-linux-arm64-musl': 1.1.0 + '@napi-rs/tar-linux-ppc64-gnu': 1.1.0 + '@napi-rs/tar-linux-s390x-gnu': 1.1.0 + '@napi-rs/tar-linux-x64-gnu': 1.1.0 + '@napi-rs/tar-linux-x64-musl': 1.1.0 + '@napi-rs/tar-wasm32-wasi': 1.1.0(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@napi-rs/tar-win32-arm64-msvc': 1.1.0 + '@napi-rs/tar-win32-ia32-msvc': 1.1.0 + '@napi-rs/tar-win32-x64-msvc': 1.1.0 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.8.1 @@ -7383,6 +8303,76 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + dependencies: + '@emnapi/core': 1.8.1 + '@emnapi/runtime': 1.8.1 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@napi-rs/wasm-tools-android-arm-eabi@1.0.1': + optional: true + + '@napi-rs/wasm-tools-android-arm64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-darwin-arm64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-darwin-x64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-freebsd-x64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-arm64-musl@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-x64-gnu@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-x64-musl@1.0.1': + optional: true + + '@napi-rs/wasm-tools-wasm32-wasi@1.0.1(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + dependencies: + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + optional: true + + '@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools-win32-x64-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools@1.0.1(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1)': + optionalDependencies: + '@napi-rs/wasm-tools-android-arm-eabi': 1.0.1 + '@napi-rs/wasm-tools-android-arm64': 1.0.1 + '@napi-rs/wasm-tools-darwin-arm64': 1.0.1 + '@napi-rs/wasm-tools-darwin-x64': 1.0.1 + '@napi-rs/wasm-tools-freebsd-x64': 1.0.1 + '@napi-rs/wasm-tools-linux-arm64-gnu': 1.0.1 + '@napi-rs/wasm-tools-linux-arm64-musl': 1.0.1 + '@napi-rs/wasm-tools-linux-x64-gnu': 1.0.1 + '@napi-rs/wasm-tools-linux-x64-musl': 1.0.1 + '@napi-rs/wasm-tools-wasm32-wasi': 1.0.1(@emnapi/core@1.8.1)(@emnapi/runtime@1.8.1) + '@napi-rs/wasm-tools-win32-arm64-msvc': 1.0.1 + '@napi-rs/wasm-tools-win32-ia32-msvc': 1.0.1 + '@napi-rs/wasm-tools-win32-x64-msvc': 1.0.1 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + '@nodable/entities@2.1.0': {} '@node-rs/jieba-android-arm-eabi@2.0.1': @@ -7714,6 +8704,69 @@ snapshots: transitivePeerDependencies: - magicast + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.11 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.3': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.11 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.11': + dependencies: + '@octokit/endpoint': 11.0.3 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + content-type: 2.0.0 + json-with-bigint: 3.5.8 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@pinojs/redact@0.4.0': {} '@pixi/colord@2.9.6': {} @@ -8252,6 +9305,11 @@ snapshots: tslib: 2.8.1 optional: true + '@tybys/wasm-util@0.10.3': + dependencies: + tslib: 2.8.1 + optional: true + '@types/better-sqlite3@7.6.13': dependencies: '@types/node': 25.2.2 @@ -8839,6 +9897,8 @@ snapshots: baseline-browser-mapping@2.9.19: {} + before-after-hook@4.0.0: {} + better-sqlite3@12.6.2: dependencies: bindings: 1.5.0 @@ -9007,6 +10067,8 @@ snapshots: character-entities-legacy@3.0.0: {} + chardet@2.2.0: {} + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -9037,6 +10099,12 @@ snapshots: string-width: 4.2.3 optional: true + cli-width@4.1.0: {} + + clipanion@4.0.0-rc.4(typanion@3.14.0): + dependencies: + typanion: 3.14.0 + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -9055,6 +10123,8 @@ snapshots: color-name@1.1.4: {} + colorette@2.0.20: {} + colortranslator@5.0.0: {} combined-stream@1.0.8: @@ -9086,6 +10156,8 @@ snapshots: content-type@1.0.5: {} + content-type@2.0.0: {} + convert-source-map@2.0.0: {} cookie-es@1.2.2: {} @@ -9376,6 +10448,8 @@ snapshots: embla-carousel@8.6.0: {} + emnapi@1.11.1: {} + emoji-regex-xs@1.0.0: {} emoji-regex@8.0.0: {} @@ -9418,6 +10492,8 @@ snapshots: has-tostringtag: 1.0.2 hasown: 2.0.2 + es-toolkit@1.49.0: {} + es6-error@4.1.1: {} esbuild@0.21.5: @@ -9742,8 +10818,18 @@ snapshots: dependencies: fast-decode-uri-component: 1.0.1 + fast-string-truncated-width@3.0.3: {} + + fast-string-width@3.0.2: + dependencies: + fast-string-truncated-width: 3.0.3 + fast-uri@3.1.0: {} + fast-wrap-ansi@0.2.2: + dependencies: + fast-string-width: 3.0.2 + fast-xml-builder@1.2.0: dependencies: path-expression-matcher: 1.5.0 @@ -10354,6 +11440,10 @@ snapshots: dependencies: argparse: 2.0.1 + js-yaml@4.3.0: + dependencies: + argparse: 2.0.1 + jsesc@3.1.0: {} json-bigint@1.0.0: @@ -10381,6 +11471,8 @@ snapshots: json-stringify-safe@5.0.1: {} + json-with-bigint@3.5.8: {} + json5@2.2.3: {} jsonfile@4.0.0: @@ -10741,6 +11833,8 @@ snapshots: muggle-string@0.4.1: {} + mute-stream@3.0.0: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -10827,6 +11921,8 @@ snapshots: obug@2.1.1: {} + obug@2.1.3: {} + ofetch@1.5.1: dependencies: destr: 2.0.5 @@ -11462,6 +12558,8 @@ snapshots: semver@7.7.4: {} + semver@7.8.5: {} + send@1.2.1: dependencies: debug: 4.4.3 @@ -11891,6 +12989,8 @@ snapshots: dependencies: safe-buffer: 5.2.1 + typanion@3.14.0: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -12001,6 +13101,8 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 + universal-user-agent@7.0.3: {} + universalify@0.1.2: {} universalify@2.0.1: {} diff --git a/scripts/bench-weflow-parser.mts b/scripts/bench-weflow-parser.mts new file mode 100644 index 00000000..ecc26d8a --- /dev/null +++ b/scripts/bench-weflow-parser.mts @@ -0,0 +1,155 @@ +/** + * Benchmark: Rust native WeFlow parser vs pure-TS stream-json parser. + * + * Generates a synthetic WeFlow export and parses it with both paths. + * + * Usage: + * pnpm exec tsx scripts/bench-weflow-parser.mts [messageCount] + * + * Requires the native module to be built first: pnpm build:native + */ + +import { createWriteStream, mkdtempSync, rmSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { once } from 'node:events' + +import { parseFileSync } from '../packages/parser/src/index' +import { loadNativeParser } from '../packages/parser/src/native/loader' + +const ENV_KEY = 'CHATLAB_DISABLE_NATIVE_PERF' +const messageCount = Number(process.argv[2] ?? 500_000) + +const MEMBER_COUNT = 200 +const TYPES = ['文本消息', '图片消息', '语音消息', '引用消息', '动画表情', '系统消息'] +const SAMPLE_TEXTS = [ + '今天天气不错，我们出去玩吧！', + '哈哈哈哈哈哈这也太好笑了', + '[图片]', + '好的，收到，明天见。', + 'This is a mixed language message with some English words 和中文混排。', + '周末有人一起打球吗？地点老地方，时间下午三点，人齐就开打。', +] + +async function generateFixture(filePath: string, count: number): Promise { + const stream = createWriteStream(filePath, { encoding: 'utf-8' }) + const write = async (chunk: string) => { + if (!stream.write(chunk)) await once(stream, 'drain') + } + + const avatars: Record = {} + for (let i = 0; i < 20; i++) { + avatars[`wxid_member_${i}`] = `data:image/jpeg;base64,${'A'.repeat(2000)}` + } + + await write( + `{"weflow":{"version":"1.0.0","exportedAt":1704164645},` + + `"session":{"wxid":"bench@chatroom","nickname":"性能测试群","displayName":"性能测试群","type":"群聊"},` + + `"avatars":${JSON.stringify(avatars)},"messages":[` + ) + + const batch: string[] = [] + for (let i = 0; i < count; i++) { + const member = i % MEMBER_COUNT + const message = { + localId: i + 1, + createTime: 1704164645 + i * 3, + formattedTime: '2024-01-02 03:04:05', + type: TYPES[i % TYPES.length], + localType: 1, + content: `${SAMPLE_TEXTS[i % SAMPLE_TEXTS.length]} #${i}`, + isSend: member === 0 ? 1 : 0, + senderUsername: `wxid_member_${member}`, + senderDisplayName: `成员${member}号`, + senderAvatarKey: `wxid_member_${member}`, + source: '', + } + batch.push(JSON.stringify(message)) + if (batch.length >= 5000) { + await write((i + 1 > batch.length ? ',' : '') + batch.join(',')) + batch.length = 0 + } + } + if (batch.length > 0) { + await write((count > batch.length ? ',' : '') + batch.join(',')) + } + await write(']}') + stream.end() + await once(stream, 'finish') +} + +interface BenchResult { + label: string + durationMs: number + messages: number + members: number + rssMb: number +} + +async function benchOnce(label: string, filePath: string, disableNative: boolean): Promise { + if (disableNative) { + process.env[ENV_KEY] = '1' + } else { + delete process.env[ENV_KEY] + } + global.gc?.() + const start = performance.now() + const result = await parseFileSync(filePath) + const durationMs = performance.now() - start + return { + label, + durationMs, + messages: result.messages.length, + members: result.members.length, + rssMb: process.memoryUsage().rss / 1024 / 1024, + } +} + +async function main() { + const saved = process.env[ENV_KEY] + delete process.env[ENV_KEY] + const nativeAvailable = loadNativeParser() !== null + if (saved !== undefined) process.env[ENV_KEY] = saved + if (!nativeAvailable) { + console.error('Native module not built. Run: pnpm build:native') + process.exit(1) + } + + const dir = mkdtempSync(join(tmpdir(), 'chatlab-weflow-bench-')) + const filePath = join(dir, 'bench.json') + try { + console.log(`Generating fixture with ${messageCount.toLocaleString()} messages...`) + await generateFixture(filePath, messageCount) + const sizeMb = statSync(filePath).size / 1024 / 1024 + console.log(`Fixture size: ${sizeMb.toFixed(1)} MB\n`) + + const results: BenchResult[] = [] + // Interleave runs to be fair about cache warmth: TS, native, TS, native. + results.push(await benchOnce('ts #1', filePath, true)) + results.push(await benchOnce('native #1', filePath, false)) + results.push(await benchOnce('ts #2', filePath, true)) + results.push(await benchOnce('native #2', filePath, false)) + + for (const r of results) { + console.log( + `${r.label.padEnd(10)} ${r.durationMs.toFixed(0).padStart(8)} ms | ` + + `${r.messages.toLocaleString()} messages, ${r.members} members | rss ${r.rssMb.toFixed(0)} MB` + ) + } + + const tsBest = Math.min(...results.filter((r) => r.label.startsWith('ts')).map((r) => r.durationMs)) + const nativeBest = Math.min(...results.filter((r) => r.label.startsWith('native')).map((r) => r.durationMs)) + console.log(`\nSpeedup (best of 2): ${(tsBest / nativeBest).toFixed(2)}x`) + + const tsMessages = results[0].messages + const nativeMessages = results[1].messages + if (tsMessages !== nativeMessages) { + console.error(`Output mismatch: ts=${tsMessages} native=${nativeMessages}`) + process.exit(1) + } + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +main()