fix(analysis): align browser word frequency totals

Compute total and unique word counts after the minimum-frequency filter while keeping them independent from the displayed top-N limit.
This commit is contained in:
digua committed 2026-07-22 09:44:17 +08:00
1 parent 915aa19a20
commit d5582f456c
2 files changed
+21 -7

No files matched your search

@@ -100,8 +100,23 @@ describe('getBrowserWordFrequency', () => {
{ word: 'project', count: 2, percentage: 33.33 },
])
assert.equal(result.totalMessages, 3)
assert.equal(result.totalWords, 7)
assert.equal(result.uniqueWords, 3)
assert.equal(result.totalWords, 6)
assert.equal(result.uniqueWords, 2)
})
it('keeps post-minCount totals when topN truncates the displayed words', () => {
const result = getBrowserWordFrequency(database, {
sessionId: 'session-one',
locale: 'en-US',
topN: 1,
minCount: 2,
posFilterMode: 'all',
enableStopwords: false,
})
assert.deepEqual(result.words, [{ word: 'hello', count: 4, percentage: 100 }])
assert.equal(result.totalWords, 6)
assert.equal(result.uniqueWords, 2)
})
it('applies member, time, excluded-word, and excluded-message filters', () => {
@@ -68,7 +68,6 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
.all(...queryParams) as Array<{ content: string }>
const excludedWords = new Set(excludeWords.map((word) => word.trim().toLocaleLowerCase(locale)).filter(Boolean))
const frequencies = new Map<string, number>()
let totalWords = 0
for (const message of messages) {
for (const rawWord of tokenize(message.content, locale)) {
@@ -76,12 +75,12 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
if (excludedWords.has(word)) continue
if (!isValidWord(word, locale, minWordLength, enableStopwords, isStopword)) continue
frequencies.set(word, (frequencies.get(word) ?? 0) + 1)
totalWords++
}
}
const ranked = [...frequencies.entries()]
.filter(([, count]) => count >= minCount)
const filteredFrequencies = [...frequencies.entries()].filter(([, count]) => count >= minCount)
const totalWords = filteredFrequencies.reduce((sum, [, count]) => sum + count, 0)
const ranked = filteredFrequencies
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0], locale))
.slice(0, Math.max(0, topN))
const rankedTotal = ranked.reduce((sum, [, count]) => sum + count, 0)
@@ -94,6 +93,6 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
})),
totalWords,
totalMessages: messages.length,
uniqueWords: frequencies.size,
uniqueWords: filteredFrequencies.length,
}
}