refactor(node-runtime): extract ftsTokenizer to @openchatlab/node-runtime

Move tokenizeForFts and tokenizeQueryForFts from electron/main/nlp/ to
packages/node-runtime/src/nlp/fts-tokenizer.ts. Uses the already-shared
getJieba() internally. Electron ftsTokenizer.ts now re-exports.
Added 9 unit tests.

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/node-runtime/src/index.ts

@@ -31,6 +31,8 @@ export {
   downloadDict,
   deleteDict,
   ensureDefaultDict,
+  tokenizeForFts,
+  tokenizeQueryForFts,
 } from './nlp'
 
 // AI 助手/技能解析器 + 对话管理

packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { tokenizeForFts, tokenizeQueryForFts } from '../fts-tokenizer'
+
+describe('tokenizeForFts', () => {
+  it('returns empty string for null/undefined/empty', () => {
+    assert.equal(tokenizeForFts(null), '')
+    assert.equal(tokenizeForFts(undefined), '')
+    assert.equal(tokenizeForFts(''), '')
+    assert.equal(tokenizeForFts('   '), '')
+  })
+
+  it('tokenizes text and lowercases tokens', () => {
+    const result = tokenizeForFts('Hello World')
+    assert.ok(result.length > 0)
+    assert.equal(result, result.toLowerCase())
+  })
+
+  it('produces non-empty output for Chinese text', () => {
+    const result = tokenizeForFts('今天天气很好')
+    assert.ok(result.length > 0)
+  })
+
+  it('handles mixed CJK and Latin text', () => {
+    const result = tokenizeForFts('Hello 你好 World')
+    assert.ok(result.includes('hello'))
+  })
+})
+
+describe('tokenizeQueryForFts', () => {
+  it('returns empty string for empty array', () => {
+    assert.equal(tokenizeQueryForFts([]), '')
+  })
+
+  it('returns empty string for array of whitespace', () => {
+    assert.equal(tokenizeQueryForFts(['  ', '']), '')
+  })
+
+  it('wraps single simple keyword in quotes', () => {
+    const result = tokenizeQueryForFts(['hello'])
+    assert.ok(result.includes('"hello"'))
+  })
+
+  it('joins multiple keywords with OR', () => {
+    const result = tokenizeQueryForFts(['hello', 'world'])
+    assert.ok(result.includes('OR'))
+  })
+
+  it('handles Chinese keywords', () => {
+    const result = tokenizeQueryForFts(['今天开心'])
+    assert.ok(result.length > 0)
+  })
+})

packages/node-runtime/src/nlp/fts-tokenizer.ts

@@ -0,0 +1,75 @@
+/**
+ * FTS5 tokenizer — shared implementation.
+ *
+ * Unlike NLP word-frequency analysis, FTS tokenization keeps all tokens
+ * (no POS filtering, no stopwords) to maximize recall in search scenarios.
+ *
+ * Uses jieba for Chinese (naturally handles mixed CJK/Latin text),
+ * falls back to whitespace split when jieba is unavailable.
+ */
+
+import { getJieba } from './segmenter'
+
+export function tokenizeForFts(text: string | null | undefined): string {
+  if (!text || text.trim().length === 0) return ''
+
+  try {
+    const jieba = getJieba()
+    const tokens = jieba.cut(text, false)
+    return tokens
+      .map((t) => t.trim().toLowerCase())
+      .filter((t) => t.length > 0)
+      .join(' ')
+  } catch {
+    return fallbackTokenize(text)
+  }
+}
+
+function fallbackTokenize(text: string): string {
+  return text
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 0)
+    .join(' ')
+}
+
+function escapeToken(token: string): string {
+  return `"${token.replace(/"/g, '""')}"`
+}
+
+/**
+ * Convert a list of user search keywords into an FTS5 MATCH expression.
+ *
+ * - Single keyword: tokens joined with AND (all must appear)
+ * - Multiple keywords: groups joined with OR (any match)
+ */
+export function tokenizeQueryForFts(keywords: string[]): string {
+  if (keywords.length === 0) return ''
+
+  const groups = keywords
+    .map((kw) => {
+      const trimmed = kw.trim()
+      if (!trimmed) return ''
+
+      try {
+        const jieba = getJieba()
+        const tokens = jieba
+          .cut(trimmed, false)
+          .map((t) => t.trim().toLowerCase())
+          .filter((t) => t.length > 0)
+
+        if (tokens.length === 0) return ''
+        if (tokens.length === 1) return escapeToken(tokens[0])
+        return tokens.map(escapeToken).join(' ')
+      } catch {
+        const simple = trimmed.toLowerCase().trim()
+        return simple ? escapeToken(simple) : ''
+      }
+    })
+    .filter((g) => g.length > 0)
+
+  if (groups.length === 0) return ''
+  if (groups.length === 1) return groups[0]
+
+  return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ')
+}

packages/node-runtime/src/nlp/index.ts

@@ -20,6 +20,9 @@ export {
 // 词频统计
 export { computeWordFrequency, segmentText } from './word-frequency'
 
+// FTS tokenizer
+export { tokenizeForFts, tokenizeQueryForFts } from './fts-tokenizer'
+
 // 词库管理
 export {
   isDictDownloaded,