mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
fix(analysis): align browser word frequency totals
Compute total and unique word counts after the minimum-frequency filter while keeping them independent from the displayed top-N limit.
This commit is contained in:
1 parent
915aa19a20
commit
d5582f456c
2 files changed
+21
-7
No files matched your search
@@ -100,8 +100,23 @@ describe('getBrowserWordFrequency', () => {
|
||||
{ word: 'project', count: 2, percentage: 33.33 },
|
||||
])
|
||||
assert.equal(result.totalMessages, 3)
|
||||
assert.equal(result.totalWords, 7)
|
||||
assert.equal(result.uniqueWords, 3)
|
||||
assert.equal(result.totalWords, 6)
|
||||
assert.equal(result.uniqueWords, 2)
|
||||
})
|
||||
|
||||
it('keeps post-minCount totals when topN truncates the displayed words', () => {
|
||||
const result = getBrowserWordFrequency(database, {
|
||||
sessionId: 'session-one',
|
||||
locale: 'en-US',
|
||||
topN: 1,
|
||||
minCount: 2,
|
||||
posFilterMode: 'all',
|
||||
enableStopwords: false,
|
||||
})
|
||||
|
||||
assert.deepEqual(result.words, [{ word: 'hello', count: 4, percentage: 100 }])
|
||||
assert.equal(result.totalWords, 6)
|
||||
assert.equal(result.uniqueWords, 2)
|
||||
})
|
||||
|
||||
it('applies member, time, excluded-word, and excluded-message filters', () => {
|
||||
|
||||
@@ -68,7 +68,6 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
|
||||
.all(...queryParams) as Array<{ content: string }>
|
||||
const excludedWords = new Set(excludeWords.map((word) => word.trim().toLocaleLowerCase(locale)).filter(Boolean))
|
||||
const frequencies = new Map<string, number>()
|
||||
let totalWords = 0
|
||||
|
||||
for (const message of messages) {
|
||||
for (const rawWord of tokenize(message.content, locale)) {
|
||||
@@ -76,12 +75,12 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
|
||||
if (excludedWords.has(word)) continue
|
||||
if (!isValidWord(word, locale, minWordLength, enableStopwords, isStopword)) continue
|
||||
frequencies.set(word, (frequencies.get(word) ?? 0) + 1)
|
||||
totalWords++
|
||||
}
|
||||
}
|
||||
|
||||
const ranked = [...frequencies.entries()]
|
||||
.filter(([, count]) => count >= minCount)
|
||||
const filteredFrequencies = [...frequencies.entries()].filter(([, count]) => count >= minCount)
|
||||
const totalWords = filteredFrequencies.reduce((sum, [, count]) => sum + count, 0)
|
||||
const ranked = filteredFrequencies
|
||||
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0], locale))
|
||||
.slice(0, Math.max(0, topN))
|
||||
const rankedTotal = ranked.reduce((sum, [, count]) => sum + count, 0)
|
||||
@@ -94,6 +93,6 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
|
||||
})),
|
||||
totalWords,
|
||||
totalMessages: messages.length,
|
||||
uniqueWords: frequencies.size,
|
||||
uniqueWords: filteredFrequencies.length,
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user