perf(nlp): avoid re-segmentation when switching wordcloud word count

Switching the word count (80/100/150/200/300) in Insights > Topics >
Wordcloud re-ran a full-corpus jieba segmentation on the Electron main
thread, since topN only affects the final slice while the segmentation
cost is independent of it. This blocked the main process and froze the UI.

- Frontend: fetch the full word list once (max tier) and slice locally on
  word-count change, recomputing percentages over the displayed subset.
  Word-count switching no longer hits the backend or triggers segmentation.
- Backend: for Chinese meaningful/custom mode, produce word frequency and
  POS-tag stats in a single jieba.tag pass instead of two, halving CPU for
  genuine recomputes (filter/time/member changes). Behavior is preserved
  (verified by an equivalence regression test).

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/node-runtime/src/nlp/segmenter.test.ts

@@ -0,0 +1,89 @@
+/**
+ * batchSegmentChineseWithStats 等价性测试
+ *
+ * 优化（C）：中文 meaningful/custom 模式下用单遍分词同时产出词频与词性统计。
+ * 必须与旧实现 batchSegmentWithFrequency + collectPosTagStats 的组合结果完全一致，
+ * 否则会改变词云与词性分布的展示行为。
+ *
+ * 运行：node --import tsx --test packages/node-runtime/src/nlp/segmenter.test.ts
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { batchSegmentChineseWithStats, batchSegmentWithFrequency, collectPosTagStats } from './segmenter'
+
+const TEXTS = [
+  '今天天气很好，我们一起去公园散步聊天',
+  '公园里有很多人在散步，天气真的非常好',
+  '我喜欢在周末去公园看书喝咖啡',
+  '北京的秋天天气最舒服，适合出去走走',
+  '聊天的时候要注意倾听对方说话',
+  '咖啡和书是我周末的最爱',
+]
+
+function assertMapEqual(actual: Map<string, number>, expected: Map<string, number>, label: string): void {
+  assert.equal(actual.size, expected.size, `${label}: size mismatch`)
+  for (const [key, value] of expected) {
+    assert.equal(actual.get(key), value, `${label}: value mismatch for "${key}"`)
+  }
+}
+
+describe('batchSegmentChineseWithStats 与旧两遍分词等价', () => {
+  it('meaningful 模式：词频/总数/去重数/词性统计均一致', () => {
+    const opts = {
+      minLength: 2,
+      minCount: 1,
+      topN: 100,
+      posFilterMode: 'meaningful' as const,
+      enableStopwords: true,
+      dictType: 'default' as const,
+    }
+    const combined = batchSegmentChineseWithStats(TEXTS, 'zh-CN', opts)
+    const freqOnly = batchSegmentWithFrequency(TEXTS, 'zh-CN', opts)
+    const posOnly = collectPosTagStats(TEXTS, 2, true, 'default')
+
+    assertMapEqual(combined.words, freqOnly.words, 'words')
+    assert.equal(combined.uniqueWords, freqOnly.uniqueWords, 'uniqueWords')
+    assert.equal(combined.totalWords, freqOnly.totalWords, 'totalWords')
+    assertMapEqual(combined.posTagStats, posOnly, 'posTagStats')
+  })
+
+  it('custom 模式 + 排除词 + minCount：结果一致', () => {
+    const opts = {
+      minLength: 2,
+      minCount: 2,
+      topN: 50,
+      posFilterMode: 'custom' as const,
+      customPosTags: ['n', 'v', 'a'],
+      enableStopwords: true,
+      dictType: 'default' as const,
+      excludeWords: ['公园'],
+    }
+    const combined = batchSegmentChineseWithStats(TEXTS, 'zh-CN', opts)
+    const freqOnly = batchSegmentWithFrequency(TEXTS, 'zh-CN', opts)
+    const posOnly = collectPosTagStats(TEXTS, 2, true, 'default')
+
+    assertMapEqual(combined.words, freqOnly.words, 'words')
+    assert.equal(combined.uniqueWords, freqOnly.uniqueWords, 'uniqueWords')
+    assert.equal(combined.totalWords, freqOnly.totalWords, 'totalWords')
+    // 词性统计不受 allowedTags / excludeWords 影响，覆盖全部有效词
+    assertMapEqual(combined.posTagStats, posOnly, 'posTagStats')
+    assert.ok(!combined.words.has('公园'), '排除词应被剔除')
+  })
+
+  it('topN 截断只影响展示数量，不影响 totalWords/uniqueWords', () => {
+    const base = {
+      minLength: 2,
+      minCount: 1,
+      posFilterMode: 'meaningful' as const,
+      enableStopwords: true,
+      dictType: 'default' as const,
+    }
+    const full = batchSegmentChineseWithStats(TEXTS, 'zh-CN', { ...base, topN: 100 })
+    const limited = batchSegmentChineseWithStats(TEXTS, 'zh-CN', { ...base, topN: 3 })
+
+    assert.ok(limited.words.size <= 3)
+    assert.equal(limited.totalWords, full.totalWords)
+    assert.equal(limited.uniqueWords, full.uniqueWords)
+  })
+})

packages/node-runtime/src/nlp/segmenter.ts

@@ -212,6 +212,68 @@ export function batchSegmentWithFrequency(
   return { words: new Map(sorted), uniqueWords: filtered.size, totalWords }
 }
 
+/**
+ * 中文单遍分词：一次 jieba.tag 同时产出词频与词性统计。
+ *
+ * 仅适用于中文 meaningful/custom 模式（all 模式走 jieba.cut，无词性）。
+ * 输出与 `batchSegmentWithFrequency` + `collectPosTagStats` 组合完全一致，
+ * 用于消除对同一批文本重复分词的开销。
+ */
+export function batchSegmentChineseWithStats(
+  texts: string[],
+  locale: SupportedLocale,
+  options: BatchSegmentOptions = {}
+): BatchSegmentResult & { posTagStats: Map<string, number> } {
+  const {
+    minLength,
+    minCount = 2,
+    topN = 100,
+    posFilterMode = 'meaningful',
+    customPosTags,
+    enableStopwords = true,
+    dictType = 'default',
+    excludeWords,
+  } = options
+
+  const effectiveMinLength = minLength ?? 2
+  const allowedTags = posFilterMode === 'custom' && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS
+  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null
+
+  const wordFrequency = new Map<string, number>()
+  const posTagStats = new Map<string, number>()
+
+  try {
+    const jieba = getJieba(dictType)
+    for (const text of texts) {
+      const cleaned = cleanText(text)
+      if (!cleaned) continue
+      for (const { tag, word } of jieba.tag(cleaned)) {
+        if (!isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword)) continue
+        // 词性统计覆盖全部有效词（与 collectPosTagStats 一致）
+        posTagStats.set(tag, (posTagStats.get(tag) || 0) + 1)
+        // 词频仅统计命中允许词性的词（与 batchSegmentWithFrequency 一致）
+        if (!allowedTags.has(tag)) continue
+        if (excludeSet && excludeSet.has(word.toLowerCase())) continue
+        wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1)
+      }
+    }
+  } catch (error) {
+    console.error('[NLP] Chinese single-pass segmentation failed:', error)
+  }
+
+  const filtered = new Map<string, number>()
+  let totalWords = 0
+  for (const [word, count] of wordFrequency) {
+    if (count >= minCount) {
+      filtered.set(word, count)
+      totalWords += count
+    }
+  }
+
+  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN)
+  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords, posTagStats }
+}
+
 /**
  * 收集文本的词性统计
  */

packages/node-runtime/src/nlp/word-frequency.ts

@@ -6,8 +6,15 @@
  */
 
 import type { DatabaseAdapter } from '@openchatlab/core'
-import type { SupportedLocale, DictType, WordFrequencyParams, WordFrequencyResult, PosTagStat } from '@openchatlab/core'
-import { segment, batchSegmentWithFrequency, collectPosTagStats } from './segmenter'
+import type {
+  SupportedLocale,
+  DictType,
+  WordFrequencyParams,
+  WordFrequencyResult,
+  PosTagStat,
+  BatchSegmentResult,
+} from '@openchatlab/core'
+import { segment, batchSegmentWithFrequency, batchSegmentChineseWithStats, collectPosTagStats } from './segmenter'
 
 function buildMessageQuery(
   timeFilter?: { startTs?: number; endTs?: number },
@@ -70,13 +77,7 @@ export function computeWordFrequency(db: DatabaseAdapter, params: WordFrequencyP
 
   const texts = messages.map((m) => m.content)
 
-  let posTagStats: PosTagStat[] | undefined
-  if (locale.startsWith('zh')) {
-    const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType as DictType)
-    posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count }))
-  }
-
-  const result = batchSegmentWithFrequency(texts, locale as SupportedLocale, {
+  const segmentOptions = {
     minLength: minWordLength,
     minCount,
     topN,
@@ -85,7 +86,23 @@ export function computeWordFrequency(db: DatabaseAdapter, params: WordFrequencyP
     enableStopwords,
     dictType: dictType as DictType,
     excludeWords,
-  })
+  }
+
+  let posTagStats: PosTagStat[] | undefined
+  let result: BatchSegmentResult
+
+  // 中文 meaningful/custom 模式下，词频与词性统计可在一次分词内同时产出，避免重复分词。
+  if (locale.startsWith('zh') && posFilterMode !== 'all') {
+    const combined = batchSegmentChineseWithStats(texts, locale as SupportedLocale, segmentOptions)
+    result = { words: combined.words, uniqueWords: combined.uniqueWords, totalWords: combined.totalWords }
+    posTagStats = [...combined.posTagStats.entries()].map(([tag, count]) => ({ tag, count }))
+  } else {
+    if (locale.startsWith('zh')) {
+      const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType as DictType)
+      posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count }))
+    }
+    result = batchSegmentWithFrequency(texts, locale as SupportedLocale, segmentOptions)
+  }
 
   let topNTotalWords = 0
   for (const count of result.words.values()) topNTotalWords += count

src/components/analysis/quotes/WordcloudTab.vue

@@ -48,7 +48,8 @@ const props = defineProps<{
 }>()
 
 const isLoading = ref(false)
-const wordcloudData = ref<EChartWordcloudData>({ words: [] })
+// 完整词表：按最大档一次性获取并缓存，切换词数时本地切片，避免重复请求与重复分词
+const allWords = ref<WordFrequencyItem[]>([])
 const stats = ref({
   totalMessages: 0,
   totalWords: 0,
@@ -64,6 +65,19 @@ const sizeScale = ref(1.25)
 // 最大显示词数（默认 150）
 const maxWords = ref(150)
 
+// 词云展示数据：从完整词表按当前词数本地切片，并按子集重算占比（纯本地计算，不触发后端分词）
+const wordcloudData = computed<EChartWordcloudData>(() => {
+  const sliced = allWords.value.slice(0, maxWords.value)
+  const total = sliced.reduce((sum, w) => sum + w.count, 0)
+  return {
+    words: sliced.map((w) => ({
+      word: w.word,
+      count: w.count,
+      percentage: total > 0 ? Math.round((w.count / total) * 10000) / 100 : 0,
+    })),
+  }
+})
+
 // 词性过滤模式
 const posFilterMode = ref<PosFilterMode>('meaningful')
 
@@ -221,6 +235,9 @@ const maxWordsOptions = [
   { label: '300', value: 300 },
 ]
 
+// 一次性获取的词数上限（取最大档）：切换词数时仅在本地切片，无需重新请求
+const MAX_WORDS = Math.max(...maxWordsOptions.map((o) => o.value))
+
 // 字体大小选项
 const sizeScaleOptions = computed(() => [
   { label: t('quotes.wordcloud.size.small'), value: 0.75 },
@@ -291,7 +308,7 @@ async function loadWordFrequency() {
       locale: locale.value,
       timeFilter: props.timeFilter ? { startTs: props.timeFilter.startTs, endTs: props.timeFilter.endTs } : undefined,
       memberId: selectedMemberId.value ?? undefined,
-      topN: maxWords.value,
+      topN: MAX_WORDS,
       minCount: 2,
       posFilterMode: posFilterMode.value,
       customPosTags: posFilterMode.value === 'custom' ? [...customPosTags.value] : undefined,
@@ -300,13 +317,11 @@ async function loadWordFrequency() {
       excludeWords: currentExcludeWords.value.length > 0 ? [...currentExcludeWords.value] : undefined,
     })
 
-    wordcloudData.value = {
-      words: result.words.map((w) => ({
-        word: w.word,
-        count: w.count,
-        percentage: w.percentage,
-      })),
-    }
+    allWords.value = result.words.map((w) => ({
+      word: w.word,
+      count: w.count,
+      percentage: w.percentage,
+    }))
 
     stats.value = {
       totalMessages: result.totalMessages,
@@ -324,7 +339,7 @@ async function loadWordFrequency() {
     }
   } catch (error) {
     console.error('加载词频数据失败:', error)
-    wordcloudData.value = { words: [] }
+    allWords.value = []
   } finally {
     isLoading.value = false
   }
@@ -338,13 +353,12 @@ watch(
   }
 )
 
-// 监听参数变化（词云主图）
+// 监听参数变化（词云主图）。注意：maxWords 不在此处——切换词数只做本地切片，不重新请求/分词。
 watch(
   () => [
     props.sessionId,
     props.timeFilter,
     selectedMemberId.value,
-    maxWords.value,
     posFilterMode.value,
     enableStopwords.value,
     selectedDictType.value,