fix(nlp): stop rebuilding jieba instance on every call without custom dict

getJieba() invalidated its cache whenever no dict file existed on disk,
so users without a custom dict rebuilt a Jieba instance per tokenize call.
FTS indexing tokenizes every message, making imports seconds slower
(43MB sample: 9.2s -> 3.5s after the fix). Cache now only invalidates
when the on-disk dict state actually changes (added or removed).

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/node-runtime/src/nlp/__tests__/segmenter-cache.test.ts

@@ -0,0 +1,55 @@
+/**
+ * Regression tests for the jieba instance cache.
+ *
+ * Bug history: when no custom dict existed on disk, every getJieba() call
+ * invalidated the cache and rebuilt a Jieba instance, making hot paths
+ * (FTS indexing tokenizes every message) rebuild jieba per call and slowing
+ * imports by seconds. The cache must only invalidate when the on-disk dict
+ * state actually changes (dict added or removed).
+ */
+
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { describe, it } from 'node:test'
+import { getJieba, initNlpDir } from '../segmenter'
+
+// Minimal valid jieba user dict: "word freq tag" lines.
+const MINIMAL_DICT = '测试词 100 n\n'
+
+describe('getJieba instance cache', () => {
+  it('reuses the cached instance across calls when no dict exists on disk', () => {
+    const nlpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-'))
+    initNlpDir(nlpDir)
+
+    const first = getJieba('zh-CN')
+    const second = getJieba('zh-CN')
+    assert.equal(second, first, 'expected the same cached jieba instance without dict changes')
+  })
+
+  it('invalidates the cache when a dict file is added, then stays stable', () => {
+    const nlpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-'))
+    initNlpDir(nlpDir)
+
+    const withoutDict = getJieba('zh-CN')
+    fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), MINIMAL_DICT)
+
+    const withDict = getJieba('zh-CN')
+    assert.notEqual(withDict, withoutDict, 'expected a rebuilt instance after dict was added')
+    assert.equal(getJieba('zh-CN'), withDict, 'expected the dict-backed instance to stay cached')
+  })
+
+  it('invalidates the cache when the dict file is removed', () => {
+    const nlpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-'))
+    initNlpDir(nlpDir)
+    fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), MINIMAL_DICT)
+
+    const withDict = getJieba('zh-CN')
+    fs.rmSync(path.join(nlpDir, 'zh-CN.dict'))
+
+    const withoutDict = getJieba('zh-CN')
+    assert.notEqual(withoutDict, withDict, 'expected a rebuilt instance after dict was removed')
+    assert.equal(getJieba('zh-CN'), withoutDict, 'expected the fallback instance to stay cached')
+  })
+})

packages/node-runtime/src/nlp/segmenter.ts

@@ -23,8 +23,14 @@ interface JiebaInstance {
   tag: (text: string) => Array<{ tag: string; word: string }>
 }
 
+interface JiebaCacheEntry {
+  instance: JiebaInstance
+  /** Whether the instance was built from a custom dict file on disk. */
+  fromDiskDict: boolean
+}
+
 let _nlpDir: string | null = null
-const jiebaInstances = new Map<DictType, JiebaInstance>()
+const jiebaInstances = new Map<DictType, JiebaCacheEntry>()
 const require = createRequire(import.meta.url)
 
 /**
@@ -61,9 +67,13 @@ export function getJieba(dictType: DictType = 'default'): JiebaInstance {
   const effectiveType = dictType === 'default' ? 'zh-CN' : dictType
   const cached = jiebaInstances.get(effectiveType)
   if (cached) {
-    if (existsDictOnDisk(effectiveType)) return cached
+    // Only invalidate when the on-disk dict state changed since the instance
+    // was built (dict added or removed). Instances built without a disk dict
+    // must stay cached, otherwise hot paths (e.g. FTS indexing) would rebuild
+    // a Jieba instance per call.
+    if (cached.fromDiskDict === existsDictOnDisk(effectiveType)) return cached.instance
     jiebaInstances.delete(effectiveType)
-    console.error(`[NLP] jieba cache invalidated (dict missing): ${effectiveType}`)
+    console.error(`[NLP] jieba cache invalidated (dict ${cached.fromDiskDict ? 'removed' : 'added'}): ${effectiveType}`)
   }
 
   try {
@@ -77,7 +87,7 @@ export function getJieba(dictType: DictType = 'default'): JiebaInstance {
       instance = new Jieba()
       console.warn(`[NLP] jieba dict missing: ${effectiveType}, fallback to built-in tokenizer`)
     }
-    jiebaInstances.set(effectiveType, instance)
+    jiebaInstances.set(effectiveType, { instance, fromDiskDict: !!diskDict })
     return instance
   } catch (error) {
     console.error(`[NLP] Failed to load jieba module (dict=${effectiveType}):`, error)