refactor(node-runtime): extract ftsTokenizer to @openchatlab/node-runtime

Move tokenizeForFts and tokenizeQueryForFts from electron/main/nlp/ to
packages/node-runtime/src/nlp/fts-tokenizer.ts. Uses the already-shared
getJieba() internally. Electron ftsTokenizer.ts now re-exports.
Added 9 unit tests.

Co-authored-by: Cursor <cursoragent@cursor.com>

electron/main/nlp/ftsTokenizer.ts

@@ -1,94 +1,7 @@
 /**
- * FTS5 专用分词器
+ * FTS tokenizer — re-exported from @openchatlab/node-runtime.
  *
- * 与 NLP 词频分析不同，FTS 分词不做词性过滤和停用词过滤，
- * 搜索场景需要保留所有词以保证召回率。
- *
- * 使用 jieba 处理中文（天然兼容中英混合文本），
- * Intl.Segmenter 处理纯英文/日文。
- *
- * 复用 segmenter 模块的 jieba 实例池，默认使用 zh-CN 词库。
+ * Kept to avoid changing internal import paths in Electron.
  */
 
-import { getJieba } from './segmenter'
-
-/**
- * 对文本进行 FTS 分词，返回空格分隔的 token 字符串。
- * 用于写入 FTS5 索引。
- */
-export function tokenizeForFts(text: string | null | undefined): string {
-  if (!text || text.trim().length === 0) return ''
-
-  try {
-    const jieba = getJieba()
-    const tokens = jieba.cut(text, false)
-    return tokens
-      .map((t) => t.trim().toLowerCase())
-      .filter((t) => t.length > 0)
-      .join(' ')
-  } catch {
-    return fallbackTokenize(text)
-  }
-}
-
-/**
- * 降级分词：jieba 不可用时按字符/空格切分
- */
-function fallbackTokenize(text: string): string {
-  return text
-    .toLowerCase()
-    .split(/\s+/)
-    .filter((t) => t.length > 0)
-    .join(' ')
-}
-
-/**
- * 转义单个 token 使其在 FTS5 MATCH 中安全使用。
- * 用双引号包裹并转义内部双引号。
- */
-function escapeToken(token: string): string {
-  return `"${token.replace(/"/g, '""')}"`
-}
-
-/**
- * 将用户搜索关键词列表转换为 FTS5 MATCH 表达式。
- *
- * 语义：
- * - 单个关键词先分词，分词结果之间为 AND（都要出现）
- * - 多个关键词之间为 OR（任一匹配即可）
- *
- * 示例：
- * - ["今天开心"]          -> '"今天" "开心"'             (AND)
- * - ["今天开心", "难过"]  -> '("今天" "开心") OR "难过"' (OR)
- * - ["hello"]             -> '"hello"'
- */
-export function tokenizeQueryForFts(keywords: string[]): string {
-  if (keywords.length === 0) return ''
-
-  const groups = keywords
-    .map((kw) => {
-      const trimmed = kw.trim()
-      if (!trimmed) return ''
-
-      try {
-        const jieba = getJieba()
-        const tokens = jieba
-          .cut(trimmed, false)
-          .map((t) => t.trim().toLowerCase())
-          .filter((t) => t.length > 0)
-
-        if (tokens.length === 0) return ''
-        if (tokens.length === 1) return escapeToken(tokens[0])
-        return tokens.map(escapeToken).join(' ')
-      } catch {
-        const simple = trimmed.toLowerCase().trim()
-        return simple ? escapeToken(simple) : ''
-      }
-    })
-    .filter((g) => g.length > 0)
-
-  if (groups.length === 0) return ''
-  if (groups.length === 1) return groups[0]
-
-  return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ')
-}
+export { tokenizeForFts, tokenizeQueryForFts } from '@openchatlab/node-runtime'

packages/node-runtime/src/index.ts

@@ -31,6 +31,8 @@ export {
   downloadDict,
   deleteDict,
   ensureDefaultDict,
+  tokenizeForFts,
+  tokenizeQueryForFts,
 } from './nlp'
 
 // AI 助手/技能解析器 + 对话管理

packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { tokenizeForFts, tokenizeQueryForFts } from '../fts-tokenizer'
+
+describe('tokenizeForFts', () => {
+  it('returns empty string for null/undefined/empty', () => {
+    assert.equal(tokenizeForFts(null), '')
+    assert.equal(tokenizeForFts(undefined), '')
+    assert.equal(tokenizeForFts(''), '')
+    assert.equal(tokenizeForFts('   '), '')
+  })
+
+  it('tokenizes text and lowercases tokens', () => {
+    const result = tokenizeForFts('Hello World')
+    assert.ok(result.length > 0)
+    assert.equal(result, result.toLowerCase())
+  })
+
+  it('produces non-empty output for Chinese text', () => {
+    const result = tokenizeForFts('今天天气很好')
+    assert.ok(result.length > 0)
+  })
+
+  it('handles mixed CJK and Latin text', () => {
+    const result = tokenizeForFts('Hello 你好 World')
+    assert.ok(result.includes('hello'))
+  })
+})
+
+describe('tokenizeQueryForFts', () => {
+  it('returns empty string for empty array', () => {
+    assert.equal(tokenizeQueryForFts([]), '')
+  })
+
+  it('returns empty string for array of whitespace', () => {
+    assert.equal(tokenizeQueryForFts(['  ', '']), '')
+  })
+
+  it('wraps single simple keyword in quotes', () => {
+    const result = tokenizeQueryForFts(['hello'])
+    assert.ok(result.includes('"hello"'))
+  })
+
+  it('joins multiple keywords with OR', () => {
+    const result = tokenizeQueryForFts(['hello', 'world'])
+    assert.ok(result.includes('OR'))
+  })
+
+  it('handles Chinese keywords', () => {
+    const result = tokenizeQueryForFts(['今天开心'])
+    assert.ok(result.length > 0)
+  })
+})

packages/node-runtime/src/nlp/fts-tokenizer.ts

@@ -0,0 +1,75 @@
+/**
+ * FTS5 tokenizer — shared implementation.
+ *
+ * Unlike NLP word-frequency analysis, FTS tokenization keeps all tokens
+ * (no POS filtering, no stopwords) to maximize recall in search scenarios.
+ *
+ * Uses jieba for Chinese (naturally handles mixed CJK/Latin text),
+ * falls back to whitespace split when jieba is unavailable.
+ */
+
+import { getJieba } from './segmenter'
+
+export function tokenizeForFts(text: string | null | undefined): string {
+  if (!text || text.trim().length === 0) return ''
+
+  try {
+    const jieba = getJieba()
+    const tokens = jieba.cut(text, false)
+    return tokens
+      .map((t) => t.trim().toLowerCase())
+      .filter((t) => t.length > 0)
+      .join(' ')
+  } catch {
+    return fallbackTokenize(text)
+  }
+}
+
+function fallbackTokenize(text: string): string {
+  return text
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 0)
+    .join(' ')
+}
+
+function escapeToken(token: string): string {
+  return `"${token.replace(/"/g, '""')}"`
+}
+
+/**
+ * Convert a list of user search keywords into an FTS5 MATCH expression.
+ *
+ * - Single keyword: tokens joined with AND (all must appear)
+ * - Multiple keywords: groups joined with OR (any match)
+ */
+export function tokenizeQueryForFts(keywords: string[]): string {
+  if (keywords.length === 0) return ''
+
+  const groups = keywords
+    .map((kw) => {
+      const trimmed = kw.trim()
+      if (!trimmed) return ''
+
+      try {
+        const jieba = getJieba()
+        const tokens = jieba
+          .cut(trimmed, false)
+          .map((t) => t.trim().toLowerCase())
+          .filter((t) => t.length > 0)
+
+        if (tokens.length === 0) return ''
+        if (tokens.length === 1) return escapeToken(tokens[0])
+        return tokens.map(escapeToken).join(' ')
+      } catch {
+        const simple = trimmed.toLowerCase().trim()
+        return simple ? escapeToken(simple) : ''
+      }
+    })
+    .filter((g) => g.length > 0)
+
+  if (groups.length === 0) return ''
+  if (groups.length === 1) return groups[0]
+
+  return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ')
+}

packages/node-runtime/src/nlp/index.ts

@@ -20,6 +20,9 @@ export {
 // 词频统计
 export { computeWordFrequency, segmentText } from './word-frequency'
 
+// FTS tokenizer
+export { tokenizeForFts, tokenizeQueryForFts } from './fts-tokenizer'
+
 // 词库管理
 export {
   isDictDownloaded,