diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59e0a7ec..ff16dd0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,6 +105,31 @@ jobs: echo "node-linker=hoisted" >> .npmrc pnpm install --frozen-lockfile + # Rust native parser：目标架构与 electron-builder --x64/--arm64 保持一致 + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.arch == 'x64' && 'x86_64-apple-darwin' || 'aarch64-apple-darwin' }} + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: packages/parser-native + key: ${{ matrix.arch }} + + - name: Build native parser (${{ matrix.arch }}) + shell: bash + run: | + if [ "${{ matrix.arch }}" == "x64" ]; then + pnpm --filter @openchatlab/parser-native build -- --target x86_64-apple-darwin + else + pnpm --filter @openchatlab/parser-native build + fi + + - name: Verify native parser artifact + shell: bash + run: ls -la packages/parser-native/chatlab-parser-native.darwin-${{ matrix.arch }}.node + - name: Inject version into app package.json shell: bash run: | @@ -151,6 +176,10 @@ jobs: working-directory: apps/desktop run: pnpm run build && pnpm exec electron-builder --mac --${{ matrix.arch }} --config electron-builder.yml -p never + - name: Verify native parser in packaged app (${{ matrix.arch }}) + shell: bash + run: node scripts/check-desktop-native.mjs darwin-${{ matrix.arch }} + - name: Upload macOS artifacts (${{ matrix.arch }}) uses: actions/upload-artifact@v6 with: @@ -202,6 +231,22 @@ jobs: echo "node-linker=hoisted" >> .npmrc pnpm install --frozen-lockfile + # Rust native parser：Windows release 仅构建 x64（与 electron-builder --win 默认一致） + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: packages/parser-native + + - name: Build native parser (win x64) + run: pnpm --filter @openchatlab/parser-native build + + - name: Verify native parser artifact + shell: bash + run: ls -la packages/parser-native/chatlab-parser-native.win32-x64-msvc.node + - name: Inject version into app package.json shell: pwsh run: | @@ -236,6 +281,10 @@ jobs: working-directory: apps/desktop run: pnpm run build && pnpm exec electron-builder --win --config electron-builder.yml -p never + - name: Verify native parser in packaged app (win x64) + shell: bash + run: node scripts/check-desktop-native.mjs win32-x64-msvc + - name: Upload Windows artifacts uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..de663daf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,75 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + - 'dev-*' + +permissions: + contents: read + +# 同一分支的旧运行自动取消，避免排队浪费 +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '24' + package-manager-cache: false + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + uses: actions/cache@v5 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: packages/parser-native + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Rust unit tests (parser-native) + working-directory: packages/parser-native + run: cargo test + + # Debug 构建足够触发 TS 侧 parity 测试（native 缺失时这些测试会自动 skip， + # 构建后即可在 CI 上真实覆盖 Rust/TS 双路径一致性）。 + - name: Build native parser (debug) + run: pnpm --filter @openchatlab/parser-native build:debug + + - name: Unit & integration tests + run: pnpm test + + - name: Type check (node) + run: pnpm run type-check:node diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index 34685744..818d8b18 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -8,7 +8,13 @@ import * as fs from 'fs' import { execSync } from 'child_process' import { Command } from 'commander' import { DEFAULT_API_PORT, loadConfig, getConfigPath, setConfigField, ConfigSetError } from '@openchatlab/config' -import { AIChatManager, initAppLogger, appLogger, getSystemLogsDir } from '@openchatlab/node-runtime' +import { + AIChatManager, + initAppLogger, + appLogger, + getSystemLogsDir, + logNativeParserStatus, +} from '@openchatlab/node-runtime' import { getVersion } from './version' import { resolveCliPath } from './paths' import { isPortAvailable, formatPortInUseError } from './http/port' @@ -63,6 +69,8 @@ program const { streamImport, detectFormat } = await import('./import') const { dbManager, pathProvider } = initRuntime() const nativeBinding = resolveNativeBinding() + // 记录 Rust native parser 可用性（本次导入是否走 Rust 内核） + logNativeParserStatus() const format = detectFormat(file) if (!format && !options.format) { diff --git a/apps/cli/src/http/index.ts b/apps/cli/src/http/index.ts index 248a2a01..fab8e229 100644 --- a/apps/cli/src/http/index.ts +++ b/apps/cli/src/http/index.ts @@ -23,6 +23,7 @@ import { createSemanticIndexWorkerRuntimeClient, initAppLogger, appLogger, + logNativeParserStatus, } from '@openchatlab/node-runtime' import type { ConfigStorage, SemanticIndexRuntime } from '@openchatlab/node-runtime' import { createServer } from './server' @@ -175,6 +176,8 @@ export async function startHttpServer(options?: HttpServerOptions): Promise<{ initAppLogger(pathProvider.getLogsDir()) initServerAiLogger(pathProvider.getLogsDir()) appLogger.info('server', `HTTP server starting on ${host}:${port}`) + // 记录 Rust native parser 可用性（导入是否走 Rust 内核），便于按日志排查回退原因 + logNativeParserStatus() setAuthToken(token) setRequireAuth(!!(options?.requireAuth ?? config.api.require_auth)) diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index fef66536..f103e825 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -29,6 +29,10 @@ files: # ${platform} -> darwin/linux/win32，${arch} -> x64/arm64，正好匹配该包目录结构。 - '!node_modules/onnxruntime-node/bin/napi-v6/**' - 'node_modules/onnxruntime-node/bin/napi-v6/${platform}/${arch}/**' + # parser-native 是 workspace 内的 Rust 包：只需要 index.js/index.d.ts/package.json 和 *.node， + # 排除 Rust 源码与构建中间产物（CI 上 target/ 可达数百 MB）。 + - '!node_modules/@openchatlab/parser-native/{target,src,node_modules}/**' + - '!node_modules/@openchatlab/parser-native/{Cargo.toml,Cargo.lock,build.rs,.gitignore}' # 哪些文件将不会被压缩，而是解压到构建目录 # 仅含 .dylib（无 .node）的原生包不会被 electron-builder 自动解包，必须显式列出， # 否则 dlopen 无法从 asar 内加载： diff --git a/apps/desktop/main/index.ts b/apps/desktop/main/index.ts index 7a8a6fcb..a329f6c4 100644 --- a/apps/desktop/main/index.ts +++ b/apps/desktop/main/index.ts @@ -27,6 +27,7 @@ import { } from './database/startup-migration' import { initLocale } from './i18n' import { MigrationRunner, ALL_MIGRATIONS } from '@openchatlab/config' +import { logNativeParserStatus } from '@openchatlab/node-runtime' import { assertDesktopDataDirCompatible, getDesktopAppVersion } from './runtime-compat' import type { RuntimeIdentity } from '@openchatlab/node-runtime/src/data-dir-compat' import { @@ -99,6 +100,8 @@ class MainProcess { async init() { initAnalytics() logger.info('Desktop app starting') + // 记录 Rust native parser 可用性（导入是否走 Rust 内核），便于按日志排查回退原因 + logNativeParserStatus() // E2E 测试模式：跳过遗留数据迁移 // 遗留迁移会删除 Documents/ChatLab，在本地测试时可能破坏用户数据 diff --git a/packages/node-runtime/src/import/index.ts b/packages/node-runtime/src/import/index.ts index a633568a..4dcd67e3 100644 --- a/packages/node-runtime/src/import/index.ts +++ b/packages/node-runtime/src/import/index.ts @@ -1,5 +1,6 @@ export { writeParseResultToDb } from './write-parse-result' export type { ImportMeta, WriteParseResultStats } from './write-parse-result' +export { logNativeParserStatus } from './native-parser-status' export { LogLevel, initPerfLog, diff --git a/packages/node-runtime/src/import/native-parser-status.ts b/packages/node-runtime/src/import/native-parser-status.ts new file mode 100644 index 00000000..833b8caa --- /dev/null +++ b/packages/node-runtime/src/import/native-parser-status.ts @@ -0,0 +1,29 @@ +/** + * Startup observability for the optional Rust parsing kernels. + * + * Desktop main process and CLI entry points call this once at startup so + * `logs/app.log` always records whether imports will use the native kernels + * or the pure-TS fallback (and why). No user data is ever logged here. + */ + +import { getNativeParserStatus } from '@openchatlab/parser' +import { appLogger } from '../logging/app-logger' + +let logged = false + +/** Log once per process whether the Rust native parser kernels can load. */ +export function logNativeParserStatus(): void { + if (logged) return + logged = true + + const status = getNativeParserStatus() + if (status.available) { + appLogger.info('import', 'Native parser kernels available (Rust module loaded)') + } else if (status.disabled) { + appLogger.info('import', 'Native parser kernels disabled via CHATLAB_DISABLE_NATIVE_PERF=1; using TS parsers') + } else { + appLogger.warn('import', 'Native parser kernels unavailable; imports fall back to TS parsers', { + reason: status.error ?? 'unknown', + }) + } +} diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index 1b84663b..70f168bc 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -15,6 +15,7 @@ export { streamParseFileInfo, analyzeIncrementalImport, incrementalImport, + logNativeParserStatus, ZipArchiveReader, validateArchiveEntryName, ArchiveImportError, diff --git a/packages/parser-native/index.d.ts b/packages/parser-native/index.d.ts index f5d70ff4..140c88c6 100644 --- a/packages/parser-native/index.d.ts +++ b/packages/parser-native/index.d.ts @@ -1,41 +1,31 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ /** - * Rust ChatLab JSON parser. Strict about the format spec: off-spec files - * error out and the JS side falls back to the pure-TS parser. - * Usage from JS: `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. + * Unified Rust parser for all native-first formats. Usage from JS: + * `parse()` → `metaJson()` → `takeMembers()` → `takeBatch()` until null → + * `summaryJson()`. Constructing with an unknown format id throws, which the + * TS wrapper treats like any other native failure (falls back to TS). */ -export declare class ChatlabParser { - constructor(filePath: string) +export declare class NativeParser { + constructor(formatId: string, filePath: string, optionsJson?: string | undefined | null) /** Run the full parse on the libuv thread pool. */ parse(): Promise /** Poll progress while `parse()` is pending. */ progress(): NativeParseProgress - getMeta(): NativeChatlabMeta + /** Format-specific meta as a JSON string (shape documented per kernel). */ + metaJson(): string /** Move members out to JS. Callable once. */ - takeMembers(): Array + takeMembers(): Array /** Move the next batch of messages out to JS; null when exhausted. */ - takeBatch(size: number): Array | null + takeBatch(size: number): Array | null + /** + * Post-parse totals as JSON: `{"messageCount":n,"memberCount":n}`. + * Valid any time after `parse()`, even once the vectors are drained. + */ + summaryJson(): string } -/** - * Rust WeFlow parser. Usage from JS: - * `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. - */ -export declare class WeflowParser { - constructor(filePath: string) - /** Run the full parse on the libuv thread pool. */ - parse(): Promise - /** Poll progress while `parse()` is pending. */ - progress(): NativeParseProgress - getMeta(): NativeWeflowMeta - /** Move members (with avatars) out to JS. Callable once. */ - takeMembers(): Array - /** Move the next batch of messages out to JS; null when exhausted. */ - takeBatch(size: number): Array | null -} - -export interface NativeChatlabMember { +export interface NativeMember { platformId: string accountName: string groupNickname?: string @@ -43,57 +33,23 @@ export interface NativeChatlabMember { roles?: Array } -export interface NativeChatlabMessage { - platformMessageId?: string - senderPlatformId: string - senderAccountName: string - senderGroupNickname?: string - timestamp: number - /** Numeric MessageType enum value from shared-types. */ - messageType: number - /** undefined here means the source `content` was JSON null. */ - content?: string - replyToMessageId?: string -} - -export interface NativeChatlabMeta { - name: string - /** Passthrough of `meta.type` ("group" when missing), matching the TS parser. */ - chatType: string - /** Passthrough of `meta.platform` ("unknown" when missing). */ - platform: string - groupId?: string - groupAvatar?: string - /** - * true when members came from the top-level `members` array; - * false when collected from messages (TS emits different object shapes). - */ - membersFromHead: boolean - messageCount: number - memberCount: number -} - export interface NativeMemberRole { id: string /** Present only when the source role object had a `name` key. */ name?: string } -export interface NativeParsedMember { - platformId: string - accountName: string - avatar?: string -} - -export interface NativeParsedMessage { - platformMessageId: string +export interface NativeMessage { + platformMessageId?: string senderPlatformId: string senderAccountName: string - /** null when the source `createTime` was JSON null (importer skips those). */ + senderGroupNickname?: string + /** None when the source timestamp was JSON null (importer skips those). */ timestamp?: number /** Numeric MessageType enum value from shared-types. */ messageType: number content?: string + replyToMessageId?: string } export interface NativeParseProgress { @@ -101,14 +57,3 @@ export interface NativeParseProgress { totalBytes: number messagesProcessed: number } - -export interface NativeWeflowMeta { - name: string - /** "group" | "private" (ChatType enum string values) */ - chatType: string - groupId?: string - groupAvatar?: string - ownerId?: string - messageCount: number - memberCount: number -} diff --git a/packages/parser-native/index.js b/packages/parser-native/index.js index 08d8921c..c6d6076b 100644 --- a/packages/parser-native/index.js +++ b/packages/parser-native/index.js @@ -587,5 +587,4 @@ if (!nativeBinding) { } module.exports = nativeBinding -module.exports.ChatlabParser = nativeBinding.ChatlabParser -module.exports.WeflowParser = nativeBinding.WeflowParser +module.exports.NativeParser = nativeBinding.NativeParser diff --git a/packages/parser-native/src/chatlab.rs b/packages/parser-native/src/chatlab.rs index 5b749f1d..029de219 100644 --- a/packages/parser-native/src/chatlab.rs +++ b/packages/parser-native/src/chatlab.rs @@ -11,50 +11,28 @@ use std::collections::HashMap; +use serde::Serialize; use serde_json::Value; +use crate::input::KernelInput; use crate::jsutil::{extract_name_from_file_path, non_empty_str}; +use crate::protocol::{KernelOutput, NativeMember, NativeMemberRole, NativeMessage}; use crate::scanner::{for_each_array_element, walk_top_level, ScanError, ScanResult}; -pub struct RoleOut { - pub id: String, - pub name: Option, -} - -pub struct ChatlabMemberOut { - pub platform_id: String, - pub account_name: String, - pub group_nickname: Option, - pub avatar: Option, - pub roles: Option>, -} - -#[derive(Default)] -pub struct ChatlabMessageOut { - pub platform_message_id: Option, - pub sender_platform_id: String, - pub sender_account_name: String, - pub sender_group_nickname: Option, - pub timestamp: f64, - pub message_type: u32, - /// None means the source `content` was JSON null. - pub content: Option, - pub reply_to_message_id: Option, -} - -pub struct ChatlabOutput { - pub name: String, +/// Shape of `metaJson()` for the chatlab kernel (consumed by the TS adapter). +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ChatlabMeta { + name: String, /// Passthrough of `meta.type` ("group" when missing/empty), like the TS parser. - pub chat_type: String, + chat_type: String, /// Passthrough of `meta.platform` ("unknown" when missing/empty). - pub platform: String, - pub group_id: Option, - pub group_avatar: Option, + platform: String, + group_id: Option, + group_avatar: Option, /// true: members came from the top-level `members` array (full shape); /// false: collected from messages (id/name/nickname only), as in TS. - pub members_from_head: bool, - pub members: Vec, - pub messages: Vec, + members_from_head: bool, } fn strict(msg: impl Into) -> ScanError { @@ -176,7 +154,7 @@ fn parse_meta(meta_raw: Option<&[u8]>, file_path: &str) -> ScanResult { }) } -fn parse_member(element: &[u8]) -> ScanResult { +fn parse_member(element: &[u8]) -> ScanResult { let value: Value = serde_json::from_slice(element).map_err(|err| strict(format!("invalid member: {err}")))?; let obj = value @@ -196,7 +174,7 @@ fn parse_member(element: &[u8]) -> ScanResult { if role.keys().any(|k| k != "id" && k != "name") { return Err(strict("member role has unsupported keys")); } - out.push(RoleOut { + out.push(NativeMemberRole { id: require_str(role, "id", "member role")?, name: optional_str(role, "name", "member role")?, }); @@ -211,7 +189,7 @@ fn parse_member(element: &[u8]) -> ScanResult { } }; - Ok(ChatlabMemberOut { + Ok(NativeMember { platform_id: require_str(obj, "platformId", "member")?, account_name: require_str(obj, "accountName", "member")?, group_nickname: optional_str(obj, "groupNickname", "member")?, @@ -255,9 +233,9 @@ impl CollectedMembers { pub fn parse_chatlab( buf: &[u8], - file_path: &str, + input: &KernelInput, mut on_progress: impl FnMut(u64, u64), -) -> ScanResult { +) -> ScanResult { let mut meta_raw: Option<&[u8]> = None; let mut members_raw: Option<&[u8]> = None; let mut messages_raw: Option<&[u8]> = None; @@ -272,9 +250,9 @@ pub fn parse_chatlab( Ok(()) })?; - let meta = parse_meta(meta_raw, file_path)?; + let meta = parse_meta(meta_raw, &input.primary_path)?; - let mut head_members: Vec = Vec::new(); + let mut head_members: Vec = Vec::new(); if let Some(raw) = members_raw { let base_offset = raw.as_ptr() as usize - buf.as_ptr() as usize; for_each_array_element(raw, base_offset, |element, _| { @@ -285,7 +263,7 @@ pub fn parse_chatlab( let members_from_head = !head_members.is_empty(); let mut collected = CollectedMembers::new(); - let mut messages: Vec = Vec::new(); + let mut messages: Vec = Vec::new(); if let Some(raw) = messages_raw { let base_offset = raw.as_ptr() as usize - buf.as_ptr() as usize; @@ -338,12 +316,12 @@ pub fn parse_chatlab( collected.observe(&sender, &account_name, group_nickname.as_ref()); } - messages.push(ChatlabMessageOut { + messages.push(NativeMessage { platform_message_id: optional_str(obj, "platformMessageId", "message")?, sender_platform_id: sender, sender_account_name: account_name, sender_group_nickname: group_nickname, - timestamp, + timestamp: Some(timestamp), message_type, content, reply_to_message_id: optional_str(obj, "replyToMessageId", "message")?, @@ -360,25 +338,28 @@ pub fn parse_chatlab( collected .order .into_iter() - .map( - |(platform_id, account_name, group_nickname)| ChatlabMemberOut { - platform_id, - account_name, - group_nickname, - avatar: None, - roles: None, - }, - ) + .map(|(platform_id, account_name, group_nickname)| NativeMember { + platform_id, + account_name, + group_nickname, + avatar: None, + roles: None, + }) .collect() }; - Ok(ChatlabOutput { + let meta_json = serde_json::to_string(&ChatlabMeta { name: meta.name, chat_type: meta.chat_type, platform: meta.platform, group_id: meta.group_id, group_avatar: meta.group_avatar, members_from_head, + }) + .map_err(|err| strict(format!("meta serialization failed: {err}")))?; + + Ok(KernelOutput { + meta_json, members, messages, }) @@ -388,8 +369,16 @@ pub fn parse_chatlab( mod tests { use super::*; - fn parse(doc: &str) -> ScanResult { - parse_chatlab(doc.as_bytes(), "/tmp/示例群.json", |_, _| {}) + fn parse(doc: &str) -> ScanResult { + let input = KernelInput { + primary_path: "/tmp/示例群.json".to_string(), + options_json: None, + }; + parse_chatlab(doc.as_bytes(), &input, |_, _| {}) + } + + fn meta(out: &KernelOutput) -> Value { + serde_json::from_str(&out.meta_json).expect("meta_json should be valid JSON") } #[test] @@ -408,11 +397,12 @@ mod tests { ] }"#; let out = parse(doc).expect("should parse"); - assert_eq!(out.name, "测试群"); - assert_eq!(out.platform, "weixin"); - assert_eq!(out.chat_type, "group"); - assert_eq!(out.group_id.as_deref(), Some("g1@chatroom")); - assert!(out.members_from_head); + let m = meta(&out); + assert_eq!(m["name"], "测试群"); + assert_eq!(m["platform"], "weixin"); + assert_eq!(m["chatType"], "group"); + assert_eq!(m["groupId"], "g1@chatroom"); + assert_eq!(m["membersFromHead"], true); assert_eq!(out.members.len(), 2); let roles = out.members[0].roles.as_ref().unwrap(); assert_eq!(roles[0].id, "owner"); @@ -422,6 +412,7 @@ mod tests { assert_eq!(out.messages.len(), 3); assert_eq!(out.messages[0].platform_message_id.as_deref(), Some("m1")); + assert_eq!(out.messages[0].timestamp, Some(100.0)); assert_eq!(out.messages[1].reply_to_message_id.as_deref(), Some("m1")); assert_eq!(out.messages[2].content, None); assert_eq!(out.messages[2].platform_message_id, None); @@ -438,7 +429,7 @@ mod tests { ]}}"# ); let out = parse(&doc).expect("should parse"); - assert!(!out.members_from_head); + assert_eq!(meta(&out)["membersFromHead"], false); assert_eq!(out.members.len(), 2); // Whole entry replaced by the later message (JS Map.set semantics): // groupNickname reset to None even though the first message had one. @@ -450,16 +441,18 @@ mod tests { #[test] fn meta_defaults_and_filename_fallback() { let out = parse(r#"{"meta": {}, "messages": []}"#).expect("should parse"); - assert_eq!(out.name, "示例群"); - assert_eq!(out.platform, "unknown"); - assert_eq!(out.chat_type, "group"); - assert_eq!(out.group_id, None); + let m = meta(&out); + assert_eq!(m["name"], "示例群"); + assert_eq!(m["platform"], "unknown"); + assert_eq!(m["chatType"], "group"); + assert_eq!(m["groupId"], Value::Null); // Explicit default name also falls back to the filename. let out2 = parse(r#"{"meta": {"name": "未知群聊", "platform": "qq"}, "messages": []}"#) .expect("should parse"); - assert_eq!(out2.name, "示例群"); - assert_eq!(out2.platform, "qq"); + let m2 = meta(&out2); + assert_eq!(m2["name"], "示例群"); + assert_eq!(m2["platform"], "qq"); // Missing meta -> strict error (TS emits a differently-shaped default // meta object, so that path must go through the fallback). @@ -493,11 +486,12 @@ mod tests { {"sender": "", "accountName": "", "timestamp": 1, "type": 0, "content": ""} ]}"#; let out = parse(doc).expect("should parse"); + let m = meta(&out); // Empty name is falsy -> default -> filename fallback; empty type -> group. - assert_eq!(out.name, "示例群"); - assert_eq!(out.chat_type, "group"); + assert_eq!(m["name"], "示例群"); + assert_eq!(m["chatType"], "group"); // groupId passes through verbatim, including empty string. - assert_eq!(out.group_id.as_deref(), Some("")); + assert_eq!(m["groupId"], ""); assert_eq!(out.messages[0].sender_platform_id, ""); assert_eq!(out.messages[0].content.as_deref(), Some("")); } diff --git a/packages/parser-native/src/input.rs b/packages/parser-native/src/input.rs new file mode 100644 index 00000000..81bda2d0 --- /dev/null +++ b/packages/parser-native/src/input.rs @@ -0,0 +1,23 @@ +//! Kernel input abstraction (first iteration). +//! +//! Today a kernel gets one primary file plus the opaque options JSON blob +//! passed to the JS constructor. Multi-file formats (QQ chunked, Google Chat +//! Takeout) will extend this struct with base-dir scoped file access. + +use std::io; + +#[derive(Clone)] +pub struct KernelInput { + pub primary_path: String, + /// Format options from the JS side (e.g. telegram `chatIndex`); no + /// current kernel consumes it yet, but it is part of the stable N-API + /// constructor signature. + #[allow(dead_code)] + pub options_json: Option, +} + +impl KernelInput { + pub fn read_primary(&self) -> io::Result> { + std::fs::read(&self.primary_path) + } +} diff --git a/packages/parser-native/src/lib.rs b/packages/parser-native/src/lib.rs index d64c26c2..87dbf847 100644 --- a/packages/parser-native/src/lib.rs +++ b/packages/parser-native/src/lib.rs @@ -1,7 +1,9 @@ #![deny(clippy::all)] mod chatlab; +mod input; mod jsutil; +mod protocol; mod scanner; mod weflow; @@ -11,31 +13,60 @@ use std::sync::{Arc, Mutex, MutexGuard}; use napi::bindgen_prelude::*; use napi_derive::napi; -use chatlab::ChatlabOutput; -use weflow::{MessageOut, WeflowOutput}; +use input::KernelInput; +use protocol::{KernelOutput, NativeMember, NativeMessage, NativeParseProgress}; -#[napi(object)] -pub struct NativeParseProgress { - pub bytes_read: f64, - pub total_bytes: f64, - pub messages_processed: f64, +// ==================== Format registry ==================== + +/// Kernel ids understood by the `NativeParser` constructor. The TS side maps +/// sniffer format ids onto these (e.g. ycccccccy-echotrace reuses "weflow"). +#[derive(Clone, Copy)] +enum FormatKind { + Weflow, + Chatlab, +} + +impl FormatKind { + fn from_id(id: &str) -> Option { + match id { + "weflow" => Some(FormatKind::Weflow), + "chatlab" => Some(FormatKind::Chatlab), + _ => None, + } + } + + fn run( + &self, + buf: &[u8], + input: &KernelInput, + on_progress: &mut dyn FnMut(u64, u64), + ) -> std::result::Result { + match self { + FormatKind::Weflow => weflow::parse_weflow(buf, input, on_progress), + FormatKind::Chatlab => chatlab::parse_chatlab(buf, input, on_progress), + } + } } // ==================== Shared parser state ==================== -struct ParsedState { - output: T, +struct ParsedState { + output: KernelOutput, cursor: usize, + /// Totals captured at parse completion; `take_*` drains the vectors, so + /// `summary_json()` reports from these instead. + message_total: usize, + member_total: usize, } -struct Shared { +struct Shared { bytes_read: AtomicU64, total_bytes: AtomicU64, messages_processed: AtomicU64, - parsed: Mutex>>, + parsed: Mutex>, } -impl Shared { +impl Shared { fn new() -> Arc { Arc::new(Shared { bytes_read: AtomicU64::new(0), @@ -53,23 +84,16 @@ impl Shared { } } - fn lock_parsed(&self) -> Result>>> { + fn lock_parsed(&self) -> Result>> { self.parsed .lock() .map_err(|_| Error::from_reason("parser state poisoned")) } - /// Run a format kernel on the file, tracking progress atomics. - fn run_parse( - &self, - file_path: &str, - kernel: impl Fn( - &[u8], - &str, - &mut dyn FnMut(u64, u64), - ) -> std::result::Result, - ) -> Result<()> { - let buf = std::fs::read(file_path) + /// Run a format kernel on the primary file, tracking progress atomics. + fn run_parse(&self, format: FormatKind, input: &KernelInput) -> Result<()> { + let buf = input + .read_primary() .map_err(|err| Error::from_reason(format!("failed to read file: {err}")))?; self.total_bytes.store(buf.len() as u64, Ordering::Relaxed); @@ -77,85 +101,52 @@ impl Shared { self.bytes_read.store(bytes, Ordering::Relaxed); self.messages_processed.store(messages, Ordering::Relaxed); }; - let output = kernel(&buf, file_path, &mut on_progress) + let output = format + .run(&buf, input, &mut on_progress) .map_err(|err| Error::from_reason(format!("parse failed: {err}")))?; self.bytes_read.store(buf.len() as u64, Ordering::Relaxed); let mut parsed = self.lock_parsed()?; - *parsed = Some(ParsedState { output, cursor: 0 }); + *parsed = Some(ParsedState { + message_total: output.messages.len(), + member_total: output.members.len(), + output, + cursor: 0, + }); Ok(()) } } -/// Take the next `size` messages out of `messages`, advancing `cursor`. -fn take_batch_from( - state: &mut ParsedState, - messages_of: impl Fn(&mut O) -> &mut Vec, - size: u32, - map: impl Fn(M) -> M, -) -> Option> { +/// Take the next `size` messages, advancing the cursor; None when exhausted. +fn take_batch_from(state: &mut ParsedState, size: u32) -> Option> { let cursor = state.cursor; - let messages = messages_of(&mut state.output); - let total = messages.len(); + let total = state.output.messages.len(); if cursor >= total { return None; } let end = (cursor + size.max(1) as usize).min(total); let mut batch = Vec::with_capacity(end - cursor); - for message in &mut messages[cursor..end] { - batch.push(map(std::mem::take(message))); + for message in &mut state.output.messages[cursor..end] { + batch.push(std::mem::take(message)); } state.cursor = end; Some(batch) } -// ==================== WeFlow ==================== +// ==================== N-API surface ==================== -#[napi(object)] -pub struct NativeWeflowMeta { - pub name: String, - /// "group" | "private" (ChatType enum string values) - pub chat_type: String, - pub group_id: Option, - pub group_avatar: Option, - pub owner_id: Option, - pub message_count: f64, - pub member_count: f64, +pub struct ParseFormatTask { + format: FormatKind, + input: KernelInput, + shared: Arc, } -#[napi(object)] -pub struct NativeParsedMember { - pub platform_id: String, - pub account_name: String, - pub avatar: Option, -} - -#[napi(object)] -pub struct NativeParsedMessage { - pub platform_message_id: String, - pub sender_platform_id: String, - pub sender_account_name: String, - /// null when the source `createTime` was JSON null (importer skips those). - pub timestamp: Option, - /// Numeric MessageType enum value from shared-types. - pub message_type: u32, - pub content: Option, -} - -pub struct ParseWeflowTask { - file_path: String, - shared: Arc>, -} - -impl Task for ParseWeflowTask { +impl Task for ParseFormatTask { type Output = (); type JsValue = (); fn compute(&mut self) -> Result { - self.shared - .run_parse(&self.file_path, |buf, path, on_progress| { - weflow::parse_weflow(buf, path, on_progress) - }) + self.shared.run_parse(self.format, &self.input) } fn resolve(&mut self, _env: Env, _output: Self::Output) -> Result { @@ -163,29 +154,39 @@ impl Task for ParseWeflowTask { } } -/// Rust WeFlow parser. Usage from JS: -/// `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. +/// Unified Rust parser for all native-first formats. Usage from JS: +/// `parse()` → `metaJson()` → `takeMembers()` → `takeBatch()` until null → +/// `summaryJson()`. Constructing with an unknown format id throws, which the +/// TS wrapper treats like any other native failure (falls back to TS). #[napi] -pub struct WeflowParser { - file_path: String, - shared: Arc>, +pub struct NativeParser { + format: FormatKind, + input: KernelInput, + shared: Arc, } #[napi] -impl WeflowParser { +impl NativeParser { #[napi(constructor)] - pub fn new(file_path: String) -> Self { - WeflowParser { - file_path, + pub fn new(format_id: String, file_path: String, options_json: Option) -> Result { + let format = FormatKind::from_id(&format_id) + .ok_or_else(|| Error::from_reason(format!("unknown native format id: {format_id}")))?; + Ok(NativeParser { + format, + input: KernelInput { + primary_path: file_path, + options_json, + }, shared: Shared::new(), - } + }) } /// Run the full parse on the libuv thread pool. #[napi] - pub fn parse(&self) -> AsyncTask { - AsyncTask::new(ParseWeflowTask { - file_path: self.file_path.clone(), + pub fn parse(&self) -> AsyncTask { + AsyncTask::new(ParseFormatTask { + format: self.format, + input: self.input.clone(), shared: Arc::clone(&self.shared), }) } @@ -196,247 +197,47 @@ impl WeflowParser { self.shared.progress() } + /// Format-specific meta as a JSON string (shape documented per kernel). #[napi] - pub fn get_meta(&self) -> Result { + pub fn meta_json(&self) -> Result { let parsed = self.shared.lock_parsed()?; let state = parsed .as_ref() - .ok_or_else(|| Error::from_reason("getMeta() called before parse() completed"))?; - Ok(NativeWeflowMeta { - name: state.output.name.clone(), - chat_type: state.output.chat_type.to_string(), - group_id: state.output.group_id.clone(), - group_avatar: state.output.group_avatar.clone(), - owner_id: state.output.owner_id.clone(), - message_count: state.output.messages.len() as f64, - member_count: state.output.members.len() as f64, - }) - } - - /// Move members (with avatars) out to JS. Callable once. - #[napi] - pub fn take_members(&self) -> Result> { - let mut parsed = self.shared.lock_parsed()?; - let state = parsed - .as_mut() - .ok_or_else(|| Error::from_reason("takeMembers() called before parse() completed"))?; - Ok(std::mem::take(&mut state.output.members) - .into_iter() - .map(|m| NativeParsedMember { - platform_id: m.platform_id, - account_name: m.account_name, - avatar: m.avatar, - }) - .collect()) - } - - /// Move the next batch of messages out to JS; null when exhausted. - #[napi] - pub fn take_batch(&self, size: u32) -> Result>> { - let mut parsed = self.shared.lock_parsed()?; - let state = parsed - .as_mut() - .ok_or_else(|| Error::from_reason("takeBatch() called before parse() completed"))?; - Ok( - take_batch_from(state, |o| &mut o.messages, size, |message| message).map(|batch| { - batch - .into_iter() - .map(|message| { - let MessageOut { - platform_message_id, - sender_platform_id, - sender_account_name, - timestamp, - message_type, - content, - } = message; - NativeParsedMessage { - platform_message_id, - sender_platform_id, - sender_account_name, - timestamp, - message_type, - content, - } - }) - .collect() - }), - ) - } -} - -// ==================== ChatLab JSON ==================== - -#[napi(object)] -pub struct NativeChatlabMeta { - pub name: String, - /// Passthrough of `meta.type` ("group" when missing), matching the TS parser. - pub chat_type: String, - /// Passthrough of `meta.platform` ("unknown" when missing). - pub platform: String, - pub group_id: Option, - pub group_avatar: Option, - /// true when members came from the top-level `members` array; - /// false when collected from messages (TS emits different object shapes). - pub members_from_head: bool, - pub message_count: f64, - pub member_count: f64, -} - -#[napi(object)] -pub struct NativeMemberRole { - pub id: String, - /// Present only when the source role object had a `name` key. - pub name: Option, -} - -#[napi(object)] -pub struct NativeChatlabMember { - pub platform_id: String, - pub account_name: String, - pub group_nickname: Option, - pub avatar: Option, - pub roles: Option>, -} - -#[napi(object)] -#[derive(Default)] -pub struct NativeChatlabMessage { - pub platform_message_id: Option, - pub sender_platform_id: String, - pub sender_account_name: String, - pub sender_group_nickname: Option, - pub timestamp: f64, - /// Numeric MessageType enum value from shared-types. - pub message_type: u32, - /// undefined here means the source `content` was JSON null. - pub content: Option, - pub reply_to_message_id: Option, -} - -pub struct ParseChatlabTask { - file_path: String, - shared: Arc>, -} - -impl Task for ParseChatlabTask { - type Output = (); - type JsValue = (); - - fn compute(&mut self) -> Result { - self.shared - .run_parse(&self.file_path, |buf, path, on_progress| { - chatlab::parse_chatlab(buf, path, on_progress) - }) - } - - fn resolve(&mut self, _env: Env, _output: Self::Output) -> Result { - Ok(()) - } -} - -/// Rust ChatLab JSON parser. Strict about the format spec: off-spec files -/// error out and the JS side falls back to the pure-TS parser. -/// Usage from JS: `parse()` → `getMeta()` → `takeMembers()` → `takeBatch()` until null. -#[napi] -pub struct ChatlabParser { - file_path: String, - shared: Arc>, -} - -#[napi] -impl ChatlabParser { - #[napi(constructor)] - pub fn new(file_path: String) -> Self { - ChatlabParser { - file_path, - shared: Shared::new(), - } - } - - /// Run the full parse on the libuv thread pool. - #[napi] - pub fn parse(&self) -> AsyncTask { - AsyncTask::new(ParseChatlabTask { - file_path: self.file_path.clone(), - shared: Arc::clone(&self.shared), - }) - } - - /// Poll progress while `parse()` is pending. - #[napi] - pub fn progress(&self) -> NativeParseProgress { - self.shared.progress() - } - - #[napi] - pub fn get_meta(&self) -> Result { - let parsed = self.shared.lock_parsed()?; - let state = parsed - .as_ref() - .ok_or_else(|| Error::from_reason("getMeta() called before parse() completed"))?; - Ok(NativeChatlabMeta { - name: state.output.name.clone(), - chat_type: state.output.chat_type.clone(), - platform: state.output.platform.clone(), - group_id: state.output.group_id.clone(), - group_avatar: state.output.group_avatar.clone(), - members_from_head: state.output.members_from_head, - message_count: state.output.messages.len() as f64, - member_count: state.output.members.len() as f64, - }) + .ok_or_else(|| Error::from_reason("metaJson() called before parse() completed"))?; + Ok(state.output.meta_json.clone()) } /// Move members out to JS. Callable once. #[napi] - pub fn take_members(&self) -> Result> { + pub fn take_members(&self) -> Result> { let mut parsed = self.shared.lock_parsed()?; let state = parsed .as_mut() .ok_or_else(|| Error::from_reason("takeMembers() called before parse() completed"))?; - Ok(std::mem::take(&mut state.output.members) - .into_iter() - .map(|m| NativeChatlabMember { - platform_id: m.platform_id, - account_name: m.account_name, - group_nickname: m.group_nickname, - avatar: m.avatar, - roles: m.roles.map(|roles| { - roles - .into_iter() - .map(|r| NativeMemberRole { - id: r.id, - name: r.name, - }) - .collect() - }), - }) - .collect()) + Ok(std::mem::take(&mut state.output.members)) } /// Move the next batch of messages out to JS; null when exhausted. #[napi] - pub fn take_batch(&self, size: u32) -> Result>> { + pub fn take_batch(&self, size: u32) -> Result>> { let mut parsed = self.shared.lock_parsed()?; let state = parsed .as_mut() .ok_or_else(|| Error::from_reason("takeBatch() called before parse() completed"))?; - Ok( - take_batch_from(state, |o| &mut o.messages, size, |m| m).map(|batch| { - batch - .into_iter() - .map(|message| NativeChatlabMessage { - platform_message_id: message.platform_message_id, - sender_platform_id: message.sender_platform_id, - sender_account_name: message.sender_account_name, - sender_group_nickname: message.sender_group_nickname, - timestamp: message.timestamp, - message_type: message.message_type, - content: message.content, - reply_to_message_id: message.reply_to_message_id, - }) - .collect() - }), - ) + Ok(take_batch_from(state, size)) + } + + /// Post-parse totals as JSON: `{"messageCount":n,"memberCount":n}`. + /// Valid any time after `parse()`, even once the vectors are drained. + #[napi] + pub fn summary_json(&self) -> Result { + let parsed = self.shared.lock_parsed()?; + let state = parsed + .as_ref() + .ok_or_else(|| Error::from_reason("summaryJson() called before parse() completed"))?; + Ok(format!( + "{{\"messageCount\":{},\"memberCount\":{}}}", + state.message_total, state.member_total + )) } } diff --git a/packages/parser-native/src/protocol.rs b/packages/parser-native/src/protocol.rs new file mode 100644 index 00000000..bbf4c663 --- /dev/null +++ b/packages/parser-native/src/protocol.rs @@ -0,0 +1,53 @@ +//! Unified kernel output protocol shared by every format kernel. +//! +//! Members and messages use one superset struct each (fields the format does +//! not produce stay `None`); the format-specific meta travels as a JSON string +//! so adding formats never changes the N-API surface. + +use napi_derive::napi; + +#[napi(object)] +pub struct NativeParseProgress { + pub bytes_read: f64, + pub total_bytes: f64, + pub messages_processed: f64, +} + +#[napi(object)] +pub struct NativeMemberRole { + pub id: String, + /// Present only when the source role object had a `name` key. + pub name: Option, +} + +#[napi(object)] +pub struct NativeMember { + pub platform_id: String, + pub account_name: String, + pub group_nickname: Option, + pub avatar: Option, + pub roles: Option>, +} + +#[napi(object)] +#[derive(Default)] +pub struct NativeMessage { + pub platform_message_id: Option, + pub sender_platform_id: String, + pub sender_account_name: String, + pub sender_group_nickname: Option, + /// None when the source timestamp was JSON null (importer skips those). + pub timestamp: Option, + /// Numeric MessageType enum value from shared-types. + pub message_type: u32, + pub content: Option, + pub reply_to_message_id: Option, +} + +/// What a format kernel returns: meta as format-specific JSON plus unified +/// member/message structs, pumped to JS through the shared `NativeParser`. +pub struct KernelOutput { + pub meta_json: String, + pub members: Vec, + pub messages: Vec, +} diff --git a/packages/parser-native/src/weflow.rs b/packages/parser-native/src/weflow.rs index 0a70923f..347700aa 100644 --- a/packages/parser-native/src/weflow.rs +++ b/packages/parser-native/src/weflow.rs @@ -6,36 +6,24 @@ use std::collections::HashMap; +use serde::Serialize; use serde_json::Value; +use crate::input::KernelInput; use crate::jsutil::{extract_name_from_file_path, non_empty_str}; -use crate::scanner::{for_each_array_element, walk_top_level, ScanResult}; +use crate::protocol::{KernelOutput, NativeMember, NativeMessage}; +use crate::scanner::{for_each_array_element, walk_top_level, ScanError, ScanResult}; -pub struct MemberOut { - pub platform_id: String, - pub account_name: String, - pub avatar: Option, -} - -#[derive(Default)] -pub struct MessageOut { - pub platform_message_id: String, - pub sender_platform_id: String, - pub sender_account_name: String, - pub timestamp: Option, - pub message_type: u32, - pub content: Option, -} - -pub struct WeflowOutput { - pub name: String, +/// Shape of `metaJson()` for the weflow kernel (consumed by the TS adapter). +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct WeflowMeta { + name: String, /// "group" | "private" — matches the ChatType enum string values. - pub chat_type: &'static str, - pub group_id: Option, - pub group_avatar: Option, - pub owner_id: Option, - pub members: Vec, - pub messages: Vec, + chat_type: &'static str, + group_id: Option, + group_avatar: Option, + owner_id: Option, } /// Message type mapping, mirroring `convertMessageType` in weflow.ts. @@ -86,7 +74,7 @@ fn is_send_equals_one(value: Option<&Value>) -> bool { } struct MemberTracker { - order: Vec, + order: Vec, index: HashMap, } @@ -110,10 +98,12 @@ impl MemberTracker { } None => { self.index.insert(platform_id.to_string(), self.order.len()); - self.order.push(MemberOut { + self.order.push(NativeMember { platform_id: platform_id.to_string(), account_name: account_name.to_string(), + group_nickname: None, avatar: avatar.cloned(), + roles: None, }); } } @@ -122,9 +112,9 @@ impl MemberTracker { pub fn parse_weflow( buf: &[u8], - file_path: &str, + input: &KernelInput, mut on_progress: impl FnMut(u64, u64), -) -> ScanResult { +) -> ScanResult { // Pass 1: structural walk of the top-level object to locate the raw spans // we care about. `scan_value` skims containers at memchr speed, so this is // cheap even when `messages` is hundreds of MB. @@ -186,7 +176,7 @@ pub fn parse_weflow( .and_then(|s| non_empty_str(s.get("nickname"))) }) .map(|s| s.to_string()) - .unwrap_or_else(|| extract_name_from_file_path(file_path, "未知聊天")); + .unwrap_or_else(|| extract_name_from_file_path(&input.primary_path, "未知聊天")); let group_id: Option = if chat_type == "group" { session_wxid.map(|s| s.to_string()) @@ -202,7 +192,7 @@ pub fn parse_weflow( // Pass 2: stream the messages array element by element. let mut members = MemberTracker::new(); - let mut messages: Vec = Vec::new(); + let mut messages: Vec = Vec::new(); let mut owner_id: Option = None; if let Some(raw) = messages_raw { @@ -283,13 +273,16 @@ pub fn parse_weflow( } }; - messages.push(MessageOut { - platform_message_id: js_string(obj.get("localId")), + messages.push(NativeMessage { + // WeFlow always emits an id string (String(localId), JS semantics). + platform_message_id: Some(js_string(obj.get("localId"))), sender_platform_id: sender.to_string(), sender_account_name: account_name.to_string(), + sender_group_nickname: None, timestamp, message_type: convert_message_type(obj.get("type").and_then(|v| v.as_str())), content, + reply_to_message_id: None, }); on_progress(end_offset as u64, messages.len() as u64); @@ -297,12 +290,20 @@ pub fn parse_weflow( })?; } - Ok(WeflowOutput { + let meta_json = serde_json::to_string(&WeflowMeta { name, chat_type, group_id, group_avatar, owner_id, + }) + .map_err(|err| ScanError { + message: format!("meta serialization failed: {err}"), + offset: 0, + })?; + + Ok(KernelOutput { + meta_json, members: members.order, messages, }) @@ -312,8 +313,16 @@ pub fn parse_weflow( mod tests { use super::*; - fn parse(doc: &str) -> WeflowOutput { - parse_weflow(doc.as_bytes(), "/tmp/测试聊天.json", |_, _| {}).expect("parse should succeed") + fn parse(doc: &str) -> KernelOutput { + let input = KernelInput { + primary_path: "/tmp/测试聊天.json".to_string(), + options_json: None, + }; + parse_weflow(doc.as_bytes(), &input, |_, _| {}).expect("parse should succeed") + } + + fn meta(out: &KernelOutput) -> Value { + serde_json::from_str(&out.meta_json).expect("meta_json should be valid JSON") } #[test] @@ -330,21 +339,19 @@ mod tests { ] }"#; let out = parse(doc); - assert_eq!(out.name, "群显示名"); - assert_eq!(out.chat_type, "group"); - assert_eq!(out.group_id.as_deref(), Some("room@chatroom")); + let meta = meta(&out); + assert_eq!(meta["name"], "群显示名"); + assert_eq!(meta["chatType"], "group"); + assert_eq!(meta["groupId"], "room@chatroom"); // session.avatar is empty (falsy) -> falls back to avatars[groupId]. - assert_eq!( - out.group_avatar.as_deref(), - Some("data:image/jpeg;base64,ROOM") - ); - assert_eq!(out.owner_id.as_deref(), Some("wxid_b")); + assert_eq!(meta["groupAvatar"], "data:image/jpeg;base64,ROOM"); + assert_eq!(meta["ownerId"], "wxid_b"); // room@chatroom sender is skipped entirely. assert_eq!(out.messages.len(), 3); assert_eq!(out.messages[0].content.as_deref(), Some("hello")); assert_eq!(out.messages[0].message_type, 0); - assert_eq!(out.messages[0].platform_message_id, "1"); + assert_eq!(out.messages[0].platform_message_id.as_deref(), Some("1")); assert_eq!(out.messages[1].message_type, 1); assert_eq!(out.messages[2].message_type, 99); assert_eq!(out.messages[2].content, None); @@ -382,7 +389,10 @@ mod tests { // Whitespace-only content is normalized to null. assert_eq!(out.messages[1].content, None); // localId missing -> String(undefined). - assert_eq!(out.messages[0].platform_message_id, "undefined"); + assert_eq!( + out.messages[0].platform_message_id.as_deref(), + Some("undefined") + ); } #[test] @@ -403,13 +413,15 @@ mod tests { fn private_chat_and_filename_fallback() { let doc = r#"{"session": {"wxid": "wxid_friend", "type": "其他"}, "messages": []}"#; let out = parse(doc); - assert_eq!(out.chat_type, "private"); - assert_eq!(out.group_id, None); - assert_eq!(out.name, "测试聊天"); + let m = meta(&out); + assert_eq!(m["chatType"], "private"); + assert_eq!(m["groupId"], Value::Null); + assert_eq!(m["name"], "测试聊天"); let out2 = parse(r#"{"messages": []}"#); - assert_eq!(out2.chat_type, "group"); - assert_eq!(out2.name, "测试聊天"); + let m2 = meta(&out2); + assert_eq!(m2["chatType"], "group"); + assert_eq!(m2["name"], "测试聊天"); } #[test] diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index 2bce887a..1186da48 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -310,6 +310,11 @@ export { FormatSniffer, createSniffer } export { getFileSize, formatFileSize, parseTimestamp, isValidYear, createProgress, readFileHeadBytes } from './utils' +// ==================== 导出 native 加载状态（供运行时启动日志使用） ==================== + +export { getNativeParserStatus } from './native/loader' +export type { NativeParserStatus } from './native/loader' + // ==================== 回调模式 API ==================== /** diff --git a/packages/parser/src/native/chatlab-native.ts b/packages/parser/src/native/chatlab-native.ts index cf22cffb..654888ef 100644 --- a/packages/parser/src/native/chatlab-native.ts +++ b/packages/parser/src/native/chatlab-native.ts @@ -1,41 +1,33 @@ /** - * Rust-accelerated ChatLab JSON parsing with transparent TS fallback. + * ChatLab JSON adapter for the unified native-first wrapper. * - * Event order and payload shapes are identical to the pure-TS parser in - * formats/chatlab.ts for spec-compliant files (verified by parity tests). - * The Rust kernel is strict about the format spec: off-spec files (wrong - * field types, missing required fields) make parse() fail, and we fall back - * to the TS parser which replicates all JS passthrough quirks. Fallback also - * happens when the native module is unavailable or disabled via - * CHATLAB_DISABLE_NATIVE_PERF=1. + * Payload shapes are identical to the pure-TS parser in formats/chatlab.ts + * for spec-compliant files (verified by parity tests). The Rust kernel is + * strict about the format spec: off-spec files (wrong field types, missing + * required fields) make parse() fail, and the wrapper falls back to the TS + * parser which replicates all JS passthrough quirks. */ import type { ChatType, MessageType } from '@openchatlab/shared-types' -import type { ChatlabParser, NativeChatlabMember, NativeChatlabMessage } from '@openchatlab/parser-native' -import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types' -import { getFileSize, createProgress } from '../utils' -import { loadNativeParser } from './loader' +import type { NativeMember, NativeMessage } from '@openchatlab/parser-native' +import type { ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import { createNativeFirstParser, type NativeFormatAdapter, type ParseGenerator } from './create-native-parser' -const PROGRESS_POLL_INTERVAL_MS = 200 - -type ParseGenerator = (options: ParseOptions) => AsyncGenerator - -function toParsedMessage(message: NativeChatlabMessage): ParsedMessage { - return { - senderPlatformId: message.senderPlatformId, - senderAccountName: message.senderAccountName, - senderGroupNickname: message.senderGroupNickname, - timestamp: message.timestamp, - type: message.messageType as MessageType, - // The kernel rejects a missing content key, so undefined here always - // means the source value was JSON null. - content: message.content ?? null, - platformMessageId: message.platformMessageId, - replyToMessageId: message.replyToMessageId, - } +/** Shape of metaJson() from the Rust chatlab kernel. */ +interface ChatlabMetaJson { + name: string + chatType: string + platform: string + groupId: string | null + groupAvatar: string | null + /** + * true when members came from the top-level `members` array; + * false when collected from messages (TS emits different object shapes). + */ + membersFromHead: boolean } -function toParsedMember(member: NativeChatlabMember, fromHead: boolean): ParsedMember { +function toParsedMember(member: NativeMember, fromHead: boolean): ParsedMember { if (!fromHead) { // Members collected from messages carry only these three keys in the TS // parser; keep the object shape identical. @@ -55,103 +47,44 @@ function toParsedMember(member: NativeChatlabMember, fromHead: boolean): ParsedM } } -async function* parseChatLabNative( - options: ParseOptions, - parser: ChatlabParser, - fallback: ParseGenerator -): AsyncGenerator { - const { filePath, batchSize = 5000, onProgress, onLog } = options +const chatlabAdapter: NativeFormatAdapter = { + formatId: 'chatlab', + label: 'ChatLab JSON', - const totalBytes = getFileSize(filePath) - const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') - yield { type: 'progress', data: initialProgress } - onProgress?.(initialProgress) - onLog?.( - 'info', - `[NativeParser] Parsing ChatLab JSON with Rust kernel, size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB` - ) + mapMeta(metaJson: unknown): ParsedMeta { + const meta = metaJson as ChatlabMetaJson + return { + name: meta.name, + platform: meta.platform, + type: meta.chatType as ChatType, + groupId: meta.groupId ?? undefined, + groupAvatar: meta.groupAvatar ?? undefined, + } + }, - const progressTimer = setInterval(() => { - const progress = parser.progress() - onProgress?.( - createProgress( - 'parsing', - progress.bytesRead, - totalBytes, - progress.messagesProcessed, - `已处理 ${progress.messagesProcessed} 条消息...` - ) - ) - }, PROGRESS_POLL_INTERVAL_MS) + mapMembers(members: NativeMember[], metaJson: unknown): ParsedMember[] { + const fromHead = (metaJson as ChatlabMetaJson).membersFromHead + return members.map((member) => toParsedMember(member, fromHead)) + }, - try { - await parser.parse() - } catch (error) { - onLog?.( - 'warn', - `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` - ) - // Only progress events have been emitted so far — safe to restart with TS. - yield* fallback(options) - return - } finally { - clearInterval(progressTimer) - } - - const nativeMeta = parser.getMeta() - const meta: ParsedMeta = { - name: nativeMeta.name, - platform: nativeMeta.platform, - type: nativeMeta.chatType as ChatType, - groupId: nativeMeta.groupId, - groupAvatar: nativeMeta.groupAvatar, - } - yield { type: 'meta', data: meta } - - const members = parser.takeMembers().map((member) => toParsedMember(member, nativeMeta.membersFromHead)) - yield { type: 'members', data: members } - - let messagesProcessed = 0 - while (true) { - const batch = parser.takeBatch(batchSize) - if (!batch) break - messagesProcessed += batch.length - yield { type: 'messages', data: batch.map(toParsedMessage) } - // The consumer typically does synchronous DB writes per batch; yield a - // macrotask so the event loop can flush pending I/O (e.g. SSE progress - // events) instead of being starved until all batches are delivered. - await new Promise((resolve) => setImmediate(resolve)) - } - - const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') - yield { type: 'progress', data: doneProgress } - onProgress?.(doneProgress) - onLog?.('info', `解析完成: ${messagesProcessed} 条消息, ${members.length} 个成员`) - - yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: members.length } } + mapMessage(message: NativeMessage): ParsedMessage { + return { + senderPlatformId: message.senderPlatformId, + senderAccountName: message.senderAccountName, + senderGroupNickname: message.senderGroupNickname, + // The kernel rejects non-number timestamps, so this is always set. + timestamp: message.timestamp as number, + type: message.messageType as MessageType, + // The kernel rejects a missing content key, so undefined here always + // means the source value was JSON null. + content: message.content ?? null, + platformMessageId: message.platformMessageId, + replyToMessageId: message.replyToMessageId, + } + }, } /** Wrap the TS ChatLab parse generator with native acceleration. */ export function withNativeChatlab(fallback: ParseGenerator): ParseGenerator { - return async function* (options: ParseOptions): AsyncGenerator { - const native = loadNativeParser() - if (!native) { - yield* fallback(options) - return - } - - let parser: ChatlabParser - try { - parser = new native.ChatlabParser(options.filePath) - } catch (error) { - options.onLog?.( - 'warn', - `[NativeParser] Failed to create Rust parser, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` - ) - yield* fallback(options) - return - } - - yield* parseChatLabNative(options, parser, fallback) - } + return createNativeFirstParser(chatlabAdapter, fallback) } diff --git a/packages/parser/src/native/create-native-parser.ts b/packages/parser/src/native/create-native-parser.ts new file mode 100644 index 00000000..f36f3092 --- /dev/null +++ b/packages/parser/src/native/create-native-parser.ts @@ -0,0 +1,136 @@ +/** + * Unified Rust-accelerated parsing with transparent TS fallback. + * + * One wrapper serves every native-first format: it loads the native module, + * constructs the shared Rust `NativeParser`, polls progress, pumps members + * and message batches, and falls back to the pure-TS parser when the native + * module is unavailable, disabled via CHATLAB_DISABLE_NATIVE_PERF=1, or the + * Rust kernel fails (kernels fail before any data event is emitted, so a TS + * restart is always safe). + * + * Per-format specifics live in a small `NativeFormatAdapter`: the Rust kernel + * id, plus mappers from the unified native structs (and the kernel's meta + * JSON) to ParseEvent payloads. Event order and payload shapes must stay + * identical to the TS parser of the same format (verified by parity tests). + */ + +import type { NativeMember, NativeMessage, NativeParser } from '@openchatlab/parser-native' +import type { ParseEvent, ParseOptions, ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import { getFileSize, createProgress } from '../utils' +import { loadNativeParser } from './loader' + +const PROGRESS_POLL_INTERVAL_MS = 200 + +export type ParseGenerator = (options: ParseOptions) => AsyncGenerator + +export interface NativeFormatAdapter { + /** Kernel id understood by the Rust NativeParser constructor. */ + formatId: string + /** Human-readable format label used in import logs. */ + label: string + /** Map the kernel's meta JSON (parsed) to the ParsedMeta event payload. */ + mapMeta: (metaJson: unknown) => ParsedMeta + /** Map native members to the ParsedMember event payload. */ + mapMembers: (members: NativeMember[], metaJson: unknown) => ParsedMember[] + /** Map one native message to the ParsedMessage event payload. */ + mapMessage: (message: NativeMessage, metaJson: unknown) => ParsedMessage +} + +async function* pumpNativeParser( + adapter: NativeFormatAdapter, + options: ParseOptions, + parser: NativeParser, + fallback: ParseGenerator +): AsyncGenerator { + const { filePath, batchSize = 5000, onProgress, onLog } = options + + const totalBytes = getFileSize(filePath) + const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') + yield { type: 'progress', data: initialProgress } + onProgress?.(initialProgress) + onLog?.( + 'info', + `[NativeParser] Parsing ${adapter.label} with Rust kernel, size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB` + ) + + const progressTimer = setInterval(() => { + const progress = parser.progress() + onProgress?.( + createProgress( + 'parsing', + progress.bytesRead, + totalBytes, + progress.messagesProcessed, + `已处理 ${progress.messagesProcessed} 条消息...` + ) + ) + }, PROGRESS_POLL_INTERVAL_MS) + + try { + await parser.parse() + } catch (error) { + onLog?.( + 'warn', + `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + // Only progress events have been emitted so far — safe to restart with TS. + yield* fallback(options) + return + } finally { + clearInterval(progressTimer) + } + + const metaJson: unknown = JSON.parse(parser.metaJson()) + yield { type: 'meta', data: adapter.mapMeta(metaJson) } + + const members = adapter.mapMembers(parser.takeMembers(), metaJson) + yield { type: 'members', data: members } + + let messagesProcessed = 0 + while (true) { + const batch = parser.takeBatch(batchSize) + if (!batch) break + messagesProcessed += batch.length + yield { type: 'messages', data: batch.map((message) => adapter.mapMessage(message, metaJson)) } + // The consumer typically does synchronous DB writes per batch; yield a + // macrotask so the event loop can flush pending I/O (e.g. SSE progress + // events) instead of being starved until all batches are delivered. + await new Promise((resolve) => setImmediate(resolve)) + } + + const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') + yield { type: 'progress', data: doneProgress } + onProgress?.(doneProgress) + onLog?.('info', `解析完成: ${messagesProcessed} 条消息, ${members.length} 个成员`) + + yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: members.length } } +} + +/** Wrap a TS parse generator with native acceleration for the given format. */ +export function createNativeFirstParser(adapter: NativeFormatAdapter, fallback: ParseGenerator): ParseGenerator { + return async function* (options: ParseOptions): AsyncGenerator { + const native = loadNativeParser() + if (!native) { + yield* fallback(options) + return + } + + let parser: NativeParser + try { + parser = new native.NativeParser( + adapter.formatId, + options.filePath, + options.formatOptions ? JSON.stringify(options.formatOptions) : undefined + ) + } catch (error) { + options.onLog?.( + 'warn', + `[NativeParser] Failed to create Rust parser, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` + ) + yield* fallback(options) + return + } + + yield* pumpNativeParser(adapter, options, parser, fallback) + } +} diff --git a/packages/parser/src/native/loader.test.ts b/packages/parser/src/native/loader.test.ts new file mode 100644 index 00000000..48b35e53 --- /dev/null +++ b/packages/parser/src/native/loader.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict' +import { mock, test } from 'node:test' + +test('treats a loadable native module without NativeParser as unavailable', async () => { + await mock.module('@openchatlab/parser-native', { + exports: { + WeflowParser: class WeflowParser {}, + }, + }) + + const { getNativeParserStatus, loadNativeParser } = await import('./loader') + + assert.equal(loadNativeParser(), null) + assert.deepEqual(getNativeParserStatus(), { + available: false, + disabled: false, + error: 'Native parser module missing NativeParser export', + }) +}) diff --git a/packages/parser/src/native/loader.ts b/packages/parser/src/native/loader.ts index 623c08ab..f6df958d 100644 --- a/packages/parser/src/native/loader.ts +++ b/packages/parser/src/native/loader.ts @@ -9,22 +9,39 @@ import { createRequire } from 'node:module' -import type { ChatlabParser, WeflowParser } from '@openchatlab/parser-native' +import type { NativeParser } from '@openchatlab/parser-native' export interface NativeParserModule { - WeflowParser: typeof WeflowParser - ChatlabParser: typeof ChatlabParser + NativeParser: typeof NativeParser +} + +/** Load-state snapshot for startup observability (see node-runtime logNativeParserStatus). */ +export interface NativeParserStatus { + available: boolean + /** true when CHATLAB_DISABLE_NATIVE_PERF=1 forces the TS parsers. */ + disabled: boolean + /** Load failure reason when unavailable (e.g. module not shipped, ABI mismatch). */ + error?: string } let cachedModule: NativeParserModule | null | undefined +let cachedLoadError: string | undefined function isNativeDisabled(): boolean { return process.env.CHATLAB_DISABLE_NATIVE_PERF === '1' } +function isNativeParserModule(value: unknown): value is NativeParserModule { + return typeof (value as { NativeParser?: unknown } | null)?.NativeParser === 'function' +} + function requireNativeModule(): NativeParserModule { const requireFn = createRequire(import.meta.url) - return requireFn('@openchatlab/parser-native') as NativeParserModule + const module = requireFn('@openchatlab/parser-native') as unknown + if (!isNativeParserModule(module)) { + throw new Error('Native parser module missing NativeParser export') + } + return module } /** @@ -37,8 +54,16 @@ export function loadNativeParser(): NativeParserModule | null { if (cachedModule !== undefined) return cachedModule try { cachedModule = requireNativeModule() - } catch { + } catch (error) { cachedModule = null + cachedLoadError = error instanceof Error ? error.message : String(error) } return cachedModule } + +/** Report whether the Rust kernels can load in this process, without throwing. */ +export function getNativeParserStatus(): NativeParserStatus { + if (isNativeDisabled()) return { available: false, disabled: true } + const module = loadNativeParser() + return { available: module !== null, disabled: false, error: module ? undefined : cachedLoadError } +} diff --git a/packages/parser/src/native/weflow-native.ts b/packages/parser/src/native/weflow-native.ts index e5e7b26a..81503e35 100644 --- a/packages/parser/src/native/weflow-native.ts +++ b/packages/parser/src/native/weflow-native.ts @@ -1,116 +1,63 @@ /** - * Rust-accelerated WeFlow parsing with transparent TS fallback. + * WeFlow adapter for the unified native-first wrapper. * - * Event order and payload shapes are identical to the pure-TS parser in - * formats/weflow.ts (verified by parity tests): progress → meta → members → - * message batches → done. Fallback happens when the native module is - * unavailable, disabled via CHATLAB_DISABLE_NATIVE_PERF=1, or fails before - * any data event has been emitted. + * Payload shapes are identical to the pure-TS parser in formats/weflow.ts + * (verified by parity tests). Shared by the ycccccccy-echotrace format module + * (same data structure, same Rust kernel). */ import { KNOWN_PLATFORMS, ChatType, type MessageType } from '@openchatlab/shared-types' -import type { NativeParsedMessage, WeflowParser } from '@openchatlab/parser-native' -import type { ParseEvent, ParseOptions, ParsedMessage, ParsedMeta } from '../types' -import { getFileSize, createProgress } from '../utils' -import { loadNativeParser } from './loader' +import type { NativeMember, NativeMessage } from '@openchatlab/parser-native' +import type { ParsedMember, ParsedMessage, ParsedMeta } from '../types' +import { createNativeFirstParser, type NativeFormatAdapter, type ParseGenerator } from './create-native-parser' -const PROGRESS_POLL_INTERVAL_MS = 200 - -type ParseGenerator = (options: ParseOptions) => AsyncGenerator - -function toParsedMessage(message: NativeParsedMessage): ParsedMessage { - return { - platformMessageId: message.platformMessageId, - senderPlatformId: message.senderPlatformId, - senderAccountName: message.senderAccountName, - // WeFlow has no separate group nickname field (same as the TS parser). - senderGroupNickname: undefined, - // The TS parser passes createTime through as-is, including null. - timestamp: (message.timestamp ?? null) as unknown as number, - type: message.messageType as MessageType, - content: message.content ?? null, - } +/** Shape of metaJson() from the Rust weflow kernel. */ +interface WeflowMetaJson { + name: string + chatType: string + groupId: string | null + groupAvatar: string | null + ownerId: string | null } -async function* parseWeFlowNative( - options: ParseOptions, - parser: WeflowParser, - fallback: ParseGenerator -): AsyncGenerator { - const { filePath, batchSize = 5000, onProgress, onLog } = options +const weflowAdapter: NativeFormatAdapter = { + formatId: 'weflow', + label: 'WeFlow export', - const totalBytes = getFileSize(filePath) - const initialProgress = createProgress('parsing', 0, totalBytes, 0, '') - yield { type: 'progress', data: initialProgress } - onProgress?.(initialProgress) - onLog?.( - 'info', - `[NativeParser] Parsing WeFlow export with Rust kernel, size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB` - ) + mapMeta(metaJson: unknown): ParsedMeta { + const meta = metaJson as WeflowMetaJson + return { + name: meta.name, + platform: KNOWN_PLATFORMS.WECHAT, + type: meta.chatType === 'private' ? ChatType.PRIVATE : ChatType.GROUP, + groupId: meta.groupId ?? undefined, + groupAvatar: meta.groupAvatar ?? undefined, + ownerId: meta.ownerId ?? undefined, + } + }, - const progressTimer = setInterval(() => { - const progress = parser.progress() - onProgress?.( - createProgress( - 'parsing', - progress.bytesRead, - totalBytes, - progress.messagesProcessed, - `已处理 ${progress.messagesProcessed} 条消息...` - ) - ) - }, PROGRESS_POLL_INTERVAL_MS) + mapMembers(members: NativeMember[]): ParsedMember[] { + return members.map((member) => ({ + platformId: member.platformId, + accountName: member.accountName, + avatar: member.avatar, + })) + }, - try { - await parser.parse() - } catch (error) { - onLog?.( - 'warn', - `[NativeParser] Rust parse failed, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` - ) - // Only progress events have been emitted so far — safe to restart with TS. - yield* fallback(options) - return - } finally { - clearInterval(progressTimer) - } - - const nativeMeta = parser.getMeta() - const meta: ParsedMeta = { - name: nativeMeta.name, - platform: KNOWN_PLATFORMS.WECHAT, - type: nativeMeta.chatType === 'private' ? ChatType.PRIVATE : ChatType.GROUP, - groupId: nativeMeta.groupId, - groupAvatar: nativeMeta.groupAvatar, - ownerId: nativeMeta.ownerId, - } - yield { type: 'meta', data: meta } - - const members = parser.takeMembers().map((member) => ({ - platformId: member.platformId, - accountName: member.accountName, - avatar: member.avatar, - })) - yield { type: 'members', data: members } - - let messagesProcessed = 0 - while (true) { - const batch = parser.takeBatch(batchSize) - if (!batch) break - messagesProcessed += batch.length - yield { type: 'messages', data: batch.map(toParsedMessage) } - // The consumer typically does synchronous DB writes per batch; yield a - // macrotask so the event loop can flush pending I/O (e.g. SSE progress - // events) instead of being starved until all batches are delivered. - await new Promise((resolve) => setImmediate(resolve)) - } - - const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '') - yield { type: 'progress', data: doneProgress } - onProgress?.(doneProgress) - onLog?.('info', `解析完成: ${messagesProcessed} 条消息, ${members.length} 个成员`) - - yield { type: 'done', data: { messageCount: messagesProcessed, memberCount: members.length } } + mapMessage(message: NativeMessage): ParsedMessage { + return { + // The kernel always emits an id string (String(localId), JS semantics). + platformMessageId: message.platformMessageId, + senderPlatformId: message.senderPlatformId, + senderAccountName: message.senderAccountName, + // WeFlow has no separate group nickname field (same as the TS parser). + senderGroupNickname: undefined, + // The TS parser passes createTime through as-is, including null. + timestamp: (message.timestamp ?? null) as unknown as number, + type: message.messageType as MessageType, + content: message.content ?? null, + } + }, } /** @@ -118,25 +65,5 @@ async function* parseWeFlowNative( * ycccccccy-echotrace format modules (they share the same data structure). */ export function withNativeWeflow(fallback: ParseGenerator): ParseGenerator { - return async function* (options: ParseOptions): AsyncGenerator { - const native = loadNativeParser() - if (!native) { - yield* fallback(options) - return - } - - let parser: WeflowParser - try { - parser = new native.WeflowParser(options.filePath) - } catch (error) { - options.onLog?.( - 'warn', - `[NativeParser] Failed to create Rust parser, falling back to TS parser: ${error instanceof Error ? error.message : String(error)}` - ) - yield* fallback(options) - return - } - - yield* parseWeFlowNative(options, parser, fallback) - } + return createNativeFirstParser(weflowAdapter, fallback) } diff --git a/scripts/check-desktop-native.mjs b/scripts/check-desktop-native.mjs new file mode 100644 index 00000000..5a267525 --- /dev/null +++ b/scripts/check-desktop-native.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/** + * Verify the packaged Electron app ships the Rust native parser binary. + * + * Usage: node scripts/check-desktop-native.mjs + * e.g. node scripts/check-desktop-native.mjs darwin-arm64 + * node scripts/check-desktop-native.mjs win32-x64-msvc + * + * Scans apps/desktop/dist for app.asar.unpacked directories and requires + * chatlab-parser-native..node to exist (non-empty) inside + * node_modules/@openchatlab/parser-native. Fails fast so a release without + * the native kernel never ships silently. + */ + +import { existsSync, readdirSync, statSync } from 'node:fs' +import { join, resolve } from 'node:path' + +const suffix = process.argv[2] +if (!suffix) { + console.error('Usage: node scripts/check-desktop-native.mjs (e.g. darwin-arm64)') + process.exit(1) +} + +const distDir = resolve(import.meta.dirname, '../apps/desktop/dist') +if (!existsSync(distDir)) { + console.error(`[check-desktop-native] dist directory not found: ${distDir}`) + process.exit(1) +} + +/** Recursively collect app.asar.unpacked directories (bounded depth). */ +function findUnpackedDirs(dir, depth = 0, out = []) { + if (depth > 6) return out + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return out + } + for (const entry of entries) { + if (!entry.isDirectory()) continue + const full = join(dir, entry.name) + if (entry.name === 'app.asar.unpacked') { + out.push(full) + } else { + findUnpackedDirs(full, depth + 1, out) + } + } + return out +} + +const unpackedDirs = findUnpackedDirs(distDir) +if (unpackedDirs.length === 0) { + console.error(`[check-desktop-native] no app.asar.unpacked directory found under ${distDir}`) + process.exit(1) +} + +const expectedFile = `chatlab-parser-native.${suffix}.node` +let ok = false +for (const dir of unpackedDirs) { + const nodePath = join(dir, 'node_modules', '@openchatlab', 'parser-native', expectedFile) + if (existsSync(nodePath) && statSync(nodePath).size > 0) { + console.log(`[check-desktop-native] OK: ${nodePath}`) + ok = true + } else { + console.error(`[check-desktop-native] MISSING: ${nodePath}`) + } +} + +if (!ok) { + console.error(`[check-desktop-native] packaged app is missing ${expectedFile}; failing the build`) + process.exit(1) +}