mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-27 02:45:26 +08:00
perf(analysis): cache analytics results on disk keyed by db file version
Repeatedly switching time filters recomputed expensive analytics/NLP queries from scratch (e.g. a 1-year word cloud took 10s+ after a detour through 5-year), because cancelling the network request cannot stop the synchronous server-side computation. Add a shared, platform-agnostic disk cache under cacheDir/query, validated by a fingerprint derived from the session DB file (mtime+size of the main file and its -wal sidecar) so any import, incremental import or member edit transparently invalidates it. Wire it through the shared HTTP routes (word-frequency plus all analytics endpoints), so CLI Web and the desktop internal server share one implementation. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
22df3bc6fe
commit
e87b359d23
10 files changed
+566
-32
No files matched your search
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* buildAnalyticsCacheKey tests.
|
||||
*
|
||||
* 运行:node --import tsx --test packages/http-routes/src/analytics-cache.test.ts
|
||||
*
|
||||
* 键的确定性直接决定缓存命中率与正确性:等价请求必须同键,不同请求必须异键。
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach, afterEach } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { buildAnalyticsCacheKey, withAnalyticsCache } from './analytics-cache'
|
||||
import type { HttpRouteContext } from './context'
|
||||
|
||||
describe('buildAnalyticsCacheKey', () => {
|
||||
it('is independent of param key order', () => {
|
||||
const a = buildAnalyticsCacheKey('wf', { startTs: 1, endTs: 2, memberId: 3 })
|
||||
const b = buildAnalyticsCacheKey('wf', { memberId: 3, endTs: 2, startTs: 1 })
|
||||
assert.equal(a, b)
|
||||
})
|
||||
|
||||
it('ignores undefined params', () => {
|
||||
const a = buildAnalyticsCacheKey('wf', { startTs: 1, memberId: undefined })
|
||||
const b = buildAnalyticsCacheKey('wf', { startTs: 1 })
|
||||
assert.equal(a, b)
|
||||
})
|
||||
|
||||
it('separates by namespace', () => {
|
||||
const a = buildAnalyticsCacheKey('catchphrase', { startTs: 1 })
|
||||
const b = buildAnalyticsCacheKey('mention', { startTs: 1 })
|
||||
assert.notEqual(a, b)
|
||||
})
|
||||
|
||||
it('distinguishes different param values', () => {
|
||||
const a = buildAnalyticsCacheKey('wf', { startTs: 1, endTs: 2 })
|
||||
const b = buildAnalyticsCacheKey('wf', { startTs: 1, endTs: 3 })
|
||||
assert.notEqual(a, b)
|
||||
})
|
||||
|
||||
it('handles nested arrays/objects deterministically', () => {
|
||||
const a = buildAnalyticsCacheKey('wf', { excludeWords: ['b', 'a'], pos: { mode: 'custom' } })
|
||||
const b = buildAnalyticsCacheKey('wf', { pos: { mode: 'custom' }, excludeWords: ['b', 'a'] })
|
||||
assert.equal(a, b)
|
||||
const c = buildAnalyticsCacheKey('wf', { excludeWords: ['a', 'b'] })
|
||||
assert.notEqual(a, c, 'array order is significant')
|
||||
})
|
||||
})
|
||||
|
||||
describe('withAnalyticsCache', () => {
|
||||
let root: string
|
||||
let dbPath: string
|
||||
|
||||
function makeCtx(): HttpRouteContext {
|
||||
return {
|
||||
pathProvider: { getCacheDir: () => path.join(root, 'cache') },
|
||||
sessionAdapter: { getDbPath: () => dbPath },
|
||||
} as unknown as HttpRouteContext
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir()
|
||||
root = fs.mkdtempSync(path.join(baseDir, 'chatlab-analytics-cache-'))
|
||||
dbPath = path.join(root, 'session.db')
|
||||
fs.writeFileSync(dbPath, 'db-v1')
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('computes once then serves the cached value on the second call', () => {
|
||||
const ctx = makeCtx()
|
||||
let calls = 0
|
||||
const compute = () => {
|
||||
calls++
|
||||
return { value: calls }
|
||||
}
|
||||
const first = withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
const second = withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
assert.equal(calls, 1)
|
||||
assert.deepEqual(first, { value: 1 })
|
||||
assert.deepEqual(second, { value: 1 })
|
||||
// 缓存确实落到了磁盘 cacheDir/query 下
|
||||
const cacheFile = path.join(root, 'cache', 'query', 's1.cache.json')
|
||||
assert.ok(fs.existsSync(cacheFile))
|
||||
})
|
||||
|
||||
it('recomputes after the db file changes (size/mtime fingerprint)', () => {
|
||||
const ctx = makeCtx()
|
||||
let calls = 0
|
||||
const compute = () => {
|
||||
calls++
|
||||
return { value: calls }
|
||||
}
|
||||
withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
fs.writeFileSync(dbPath, 'db-v2-with-more-bytes') // 改变文件大小 => 版本变化
|
||||
const after = withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(after, { value: 2 })
|
||||
})
|
||||
|
||||
it('isolates entries by params and by session', () => {
|
||||
const ctx = makeCtx()
|
||||
let calls = 0
|
||||
const compute = () => {
|
||||
calls++
|
||||
return { value: calls }
|
||||
}
|
||||
withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
withAnalyticsCache(ctx, 's1', 'nlp.word-frequency', { startTs: 2 }, compute)
|
||||
withAnalyticsCache(ctx, 's2', 'nlp.word-frequency', { startTs: 1 }, compute)
|
||||
assert.equal(calls, 3)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Shared analytics result caching for HTTP route handlers.
|
||||
*
|
||||
* Wraps expensive analytics / NLP computations with the platform-agnostic
|
||||
* analytics cache (cacheDir/query/{sessionId}.cache.json). Used by both CLI Web
|
||||
* (`chatlab start`) and the Electron internal server so the two share one cache
|
||||
* implementation. Validity is keyed to the session DB file fingerprint, so any
|
||||
* import / incremental import / member edit transparently invalidates entries.
|
||||
*/
|
||||
|
||||
import * as path from 'path'
|
||||
import { getDbFileVersion, getOrComputeAnalysisCache } from '@openchatlab/node-runtime'
|
||||
import type { HttpRouteContext } from './context'
|
||||
|
||||
/** Deterministic stringify: sorts object keys recursively and drops `undefined`. */
|
||||
function canonical(value: unknown): string {
|
||||
if (value === undefined) return 'null'
|
||||
if (value === null || typeof value !== 'object') return JSON.stringify(value)
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`
|
||||
const obj = value as Record<string, unknown>
|
||||
const keys = Object.keys(obj)
|
||||
.filter((k) => obj[k] !== undefined)
|
||||
.sort()
|
||||
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical(obj[k])}`).join(',')}}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a stable cache key from a namespace (endpoint id) and its params.
|
||||
* Key order does not matter and `undefined` params are ignored, so equivalent
|
||||
* requests map to the same key.
|
||||
*/
|
||||
export function buildAnalyticsCacheKey(namespace: string, params: Record<string, unknown>): string {
|
||||
return `${namespace}:${canonical(params)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache-first wrapper for an analytics computation bound to a session.
|
||||
* Returns the cached result when the DB file is unchanged; otherwise computes,
|
||||
* persists (tagged with the current DB fingerprint) and returns it.
|
||||
*/
|
||||
export function withAnalyticsCache<T>(
|
||||
ctx: HttpRouteContext,
|
||||
sessionId: string,
|
||||
namespace: string,
|
||||
params: Record<string, unknown>,
|
||||
compute: () => T
|
||||
): T {
|
||||
const queryCacheDir = path.join(ctx.pathProvider.getCacheDir(), 'query')
|
||||
const version = getDbFileVersion(ctx.sessionAdapter.getDbPath(sessionId))
|
||||
const key = buildAnalyticsCacheKey(namespace, params)
|
||||
return getOrComputeAnalysisCache(sessionId, key, queryCacheDir, version, compute)
|
||||
}
|
||||
@@ -18,6 +18,7 @@ export {
|
||||
errorResponse,
|
||||
} from './errors'
|
||||
export { parseTimeFilter } from './helpers'
|
||||
export { buildAnalyticsCacheKey, withAnalyticsCache } from './analytics-cache'
|
||||
|
||||
// Individual route registration for granular testing or selective registration
|
||||
export { registerSystemRoutes } from './routes/system'
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* analytics 路由缓存接入集成测试。
|
||||
*
|
||||
* 运行:node --import tsx --test packages/http-routes/src/routes/web/analytics.test.ts
|
||||
*
|
||||
* 验证目标(接入契约,不重复 core 的算法矩阵):
|
||||
* 1. 同一查询二次请求命中磁盘缓存、不再访问数据库(命中后关闭 db 仍能成功返回)。
|
||||
* 2. DB 文件状态变化(mtime/size)使缓存失效,触发重新计算。
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach, afterEach } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import Database from 'better-sqlite3'
|
||||
import Fastify, { type FastifyInstance } from 'fastify'
|
||||
import type { DatabaseAdapter, PathProvider, PreparedStatement, RunResult } from '@openchatlab/core'
|
||||
import type { SessionRuntimeAdapter, DatabaseManager } from '@openchatlab/node-runtime'
|
||||
import type { HttpRouteContext } from '../../context'
|
||||
import { registerAnalyticsRoutes } from './analytics'
|
||||
|
||||
class Stmt implements PreparedStatement {
|
||||
readonly?: boolean
|
||||
constructor(private stmt: Database.Statement) {
|
||||
this.readonly = stmt.readonly
|
||||
}
|
||||
get(...p: unknown[]) {
|
||||
return this.stmt.get(...p) as Record<string, unknown> | undefined
|
||||
}
|
||||
all(...p: unknown[]) {
|
||||
return this.stmt.all(...p) as Record<string, unknown>[]
|
||||
}
|
||||
run(...p: unknown[]): RunResult {
|
||||
const r = this.stmt.run(...p)
|
||||
return { changes: r.changes, lastInsertRowid: r.lastInsertRowid }
|
||||
}
|
||||
}
|
||||
|
||||
class Adapter implements DatabaseAdapter {
|
||||
constructor(private db: Database.Database) {}
|
||||
exec(sql: string) {
|
||||
this.db.exec(sql)
|
||||
}
|
||||
prepare(sql: string) {
|
||||
return new Stmt(this.db.prepare(sql))
|
||||
}
|
||||
transaction<T>(fn: () => T): T {
|
||||
return this.db.transaction(fn)()
|
||||
}
|
||||
pragma(p: string) {
|
||||
return this.db.pragma(p)
|
||||
}
|
||||
close() {
|
||||
this.db.close()
|
||||
}
|
||||
}
|
||||
|
||||
const SESSION_ID = 'chat-1'
|
||||
const MEMBER_ACTIVITY_URL = `/_web/sessions/${SESSION_ID}/stats/member-activity`
|
||||
|
||||
describe('analytics routes caching', () => {
|
||||
let root: string
|
||||
let dbFile: string
|
||||
let raw: Database.Database
|
||||
let app: FastifyInstance
|
||||
|
||||
beforeEach(async () => {
|
||||
const base = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir()
|
||||
root = fs.mkdtempSync(path.join(base, 'chatlab-analytics-routes-'))
|
||||
dbFile = path.join(root, `${SESSION_ID}.db`)
|
||||
raw = new Database(dbFile)
|
||||
raw.exec(`
|
||||
CREATE TABLE member (id INTEGER PRIMARY KEY, platform_id TEXT, account_name TEXT, group_nickname TEXT, avatar TEXT);
|
||||
CREATE TABLE message (id INTEGER PRIMARY KEY, sender_id INTEGER, ts INTEGER, type INTEGER, content TEXT, platform_message_id TEXT);
|
||||
INSERT INTO member (id, platform_id, account_name) VALUES (1, 'alice', 'Alice'), (2, 'bob', 'Bob');
|
||||
INSERT INTO message (id, sender_id, ts, type, content, platform_message_id) VALUES
|
||||
(1, 1, 100, 0, 'a', 'm-1'), (2, 2, 200, 0, 'b', 'm-2'), (3, 1, 300, 0, 'c', 'm-3');
|
||||
`)
|
||||
const adapter = new Adapter(raw)
|
||||
|
||||
const pathProvider = {
|
||||
getCacheDir: () => path.join(root, 'cache'),
|
||||
} as unknown as PathProvider
|
||||
const sessionAdapter = {
|
||||
ensureReadonly: () => adapter,
|
||||
getDbPath: () => dbFile,
|
||||
} as unknown as SessionRuntimeAdapter
|
||||
const ctx = {
|
||||
sessionAdapter,
|
||||
pathProvider,
|
||||
dbManager: {} as unknown as DatabaseManager,
|
||||
getVersion: () => 'test',
|
||||
} as HttpRouteContext
|
||||
|
||||
app = Fastify()
|
||||
registerAnalyticsRoutes(app, ctx)
|
||||
await app.ready()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close()
|
||||
try {
|
||||
raw.close()
|
||||
} catch {
|
||||
/* already closed by a test */
|
||||
}
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('serves the second identical request from cache without touching the db', async () => {
|
||||
const first = await app.inject({ method: 'GET', url: MEMBER_ACTIVITY_URL })
|
||||
assert.equal(first.statusCode, 200)
|
||||
const firstBody = first.json()
|
||||
assert.ok(Array.isArray(firstBody) && firstBody.length === 2)
|
||||
|
||||
// 缓存文件已写入 cacheDir/query
|
||||
assert.ok(fs.existsSync(path.join(root, 'cache', 'query', `${SESSION_ID}.cache.json`)))
|
||||
|
||||
// 关闭底层数据库:若二次请求重新计算必然抛错,命中缓存才能成功
|
||||
raw.close()
|
||||
|
||||
const second = await app.inject({ method: 'GET', url: MEMBER_ACTIVITY_URL })
|
||||
assert.equal(second.statusCode, 200)
|
||||
assert.deepEqual(second.json(), firstBody)
|
||||
})
|
||||
|
||||
it('recomputes after the db file changes (cache invalidation)', async () => {
|
||||
const first = await app.inject({ method: 'GET', url: MEMBER_ACTIVITY_URL })
|
||||
assert.equal(first.statusCode, 200)
|
||||
|
||||
// 改变 DB 文件大小 => 版本指纹变化 => 缓存失效。配合关闭连接,重算会抛错。
|
||||
raw.close()
|
||||
fs.appendFileSync(dbFile, Buffer.from('xx'))
|
||||
|
||||
const second = await app.inject({ method: 'GET', url: MEMBER_ACTIVITY_URL })
|
||||
assert.equal(second.statusCode, 500)
|
||||
})
|
||||
})
|
||||
@@ -21,12 +21,17 @@ import {
|
||||
getLanguagePreferenceAnalysis,
|
||||
} from '@openchatlab/core'
|
||||
import { parseTimeFilter } from '../../helpers'
|
||||
import { withAnalyticsCache } from '../../analytics-cache'
|
||||
|
||||
type FilteredQuery = { startTs?: string; endTs?: string; memberId?: string }
|
||||
|
||||
export function registerAnalyticsRoutes(server: FastifyInstance, ctx: HttpRouteContext): void {
|
||||
const { sessionAdapter: adapter } = ctx
|
||||
|
||||
/** Cache-first wrapper bound to this context; `params` must capture all inputs that affect the result. */
|
||||
const cached = <T>(name: string, sessionId: string, params: Record<string, unknown>, compute: () => T): T =>
|
||||
withAnalyticsCache(ctx, sessionId, `analytics.${name}`, params, compute)
|
||||
|
||||
server.get<{ Params: { id: string } }>('/_web/sessions/:id/years', async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getAvailableYears(db)
|
||||
@@ -40,125 +45,147 @@ export function registerAnalyticsRoutes(server: FastifyInstance, ctx: HttpRouteC
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/stats/member-activity',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMemberActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('member-activity', id, { ...filter }, () => getMemberActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/stats/hourly',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getHourlyActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('hourly', id, { ...filter }, () => getHourlyActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/stats/daily',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getDailyActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('daily', id, { ...filter }, () => getDailyActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/stats/weekday',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getWeekdayActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('weekday', id, { ...filter }, () => getWeekdayActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/stats/message-types',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMessageTypeStats(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('message-types', id, { ...filter }, () => getMessageTypeStats(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/relationship',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getRelationshipStats(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('relationship', id, { ...filter }, () => getRelationshipStats(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/catchphrase',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getCatchphraseAnalysis(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('catchphrase', id, { ...filter }, () => getCatchphraseAnalysis(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/mention',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMentionAnalysis(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('mention', id, { ...filter }, () => getMentionAnalysis(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/mention-graph',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMentionGraph(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('mention-graph', id, { ...filter }, () => getMentionGraph(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/laugh',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getLaughAnalysis(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('laugh', id, { ...filter }, () => getLaughAnalysis(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery & { topEdges?: string } }>(
|
||||
'/_web/sessions/:id/analytics/cluster',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
const topEdges = request.query.topEdges ? parseInt(request.query.topEdges, 10) : undefined
|
||||
return getClusterGraph(db, filter, topEdges ? { topEdges } : undefined)
|
||||
return cached('cluster', id, { ...filter, topEdges }, () =>
|
||||
getClusterGraph(adapter.ensureReadonly(id), filter, topEdges ? { topEdges } : undefined)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery & { locale?: string } }>(
|
||||
'/_web/sessions/:id/analytics/language-preference',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
const locale = request.query.locale || 'zh-CN'
|
||||
const nlpProvider = createJiebaNlpProvider()
|
||||
return getLanguagePreferenceAnalysis(db, { locale, timeFilter: filter, nlpProvider })
|
||||
return cached('language-preference', id, { ...filter, locale }, () =>
|
||||
getLanguagePreferenceAnalysis(adapter.ensureReadonly(id), {
|
||||
locale,
|
||||
timeFilter: filter,
|
||||
nlpProvider: createJiebaNlpProvider(),
|
||||
})
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/monthly-activity',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMonthlyActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('monthly-activity', id, { ...filter }, () => getMonthlyActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/yearly-activity',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getYearlyActivity(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('yearly-activity', id, { ...filter }, () => getYearlyActivity(adapter.ensureReadonly(id), filter))
|
||||
}
|
||||
)
|
||||
|
||||
server.get<{ Params: { id: string }; Querystring: FilteredQuery }>(
|
||||
'/_web/sessions/:id/analytics/message-length-distribution',
|
||||
async (request) => {
|
||||
const db = adapter.ensureReadonly(request.params.id)
|
||||
return getMessageLengthDistribution(db, parseTimeFilter(request.query))
|
||||
const id = request.params.id
|
||||
const filter = parseTimeFilter(request.query)
|
||||
return cached('message-length-distribution', id, { ...filter }, () =>
|
||||
getMessageLengthDistribution(adapter.ensureReadonly(id), filter)
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
import * as path from 'path'
|
||||
import type { FastifyInstance } from 'fastify'
|
||||
import type { HttpRouteContext } from '../../context'
|
||||
import { withAnalyticsCache } from '../../analytics-cache'
|
||||
import type { WordFrequencyParams, SupportedLocale } from '@openchatlab/core'
|
||||
import { POS_TAG_DEFINITIONS } from '@openchatlab/core'
|
||||
import {
|
||||
@@ -52,7 +53,8 @@ export function registerNlpRoutes(server: FastifyInstance, ctx: HttpRouteContext
|
||||
if (!db) {
|
||||
throw Object.assign(new Error(`Session not found: ${params.sessionId}`), { statusCode: 404 })
|
||||
}
|
||||
return computeWordFrequency(db, params)
|
||||
const { sessionId, ...keyParams } = params
|
||||
return withAnalyticsCache(ctx, sessionId, 'nlp.word-frequency', keyParams, () => computeWordFrequency(db, params))
|
||||
})
|
||||
|
||||
server.post<{ Body: { text: string; locale: SupportedLocale; minLength?: number } }>(
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Analytics result cache tests.
|
||||
*
|
||||
* 运行:node --import tsx --test packages/node-runtime/src/cache/analytics-cache.test.ts
|
||||
*
|
||||
* 覆盖:版本指纹派生、命中/未命中、版本变更后重算覆盖、损坏缓存兜底。
|
||||
* 这些是 CLI Web 与桌面端共享的分析结果缓存核心,失效一旦错误会导致脏数据展示,必须回归。
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach, afterEach } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { getDbFileVersion, getOrComputeAnalysisCache } from './analytics-cache'
|
||||
|
||||
let tmpDir: string
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-cache-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('getDbFileVersion', () => {
|
||||
it('changes when the DB file size changes', () => {
|
||||
const dbPath = path.join(tmpDir, 's1.db')
|
||||
fs.writeFileSync(dbPath, 'a')
|
||||
const v1 = getDbFileVersion(dbPath)
|
||||
|
||||
fs.writeFileSync(dbPath, 'aa')
|
||||
const v2 = getDbFileVersion(dbPath)
|
||||
|
||||
assert.notEqual(v1, v2, 'size change must change the version')
|
||||
})
|
||||
|
||||
it('changes when the -wal sidecar appears or grows', () => {
|
||||
const dbPath = path.join(tmpDir, 's2.db')
|
||||
fs.writeFileSync(dbPath, 'main')
|
||||
const v1 = getDbFileVersion(dbPath)
|
||||
|
||||
fs.writeFileSync(`${dbPath}-wal`, 'wal-bytes')
|
||||
const v2 = getDbFileVersion(dbPath)
|
||||
|
||||
assert.notEqual(v1, v2, 'wal sidecar change must change the version')
|
||||
})
|
||||
|
||||
it('is stable when nothing changes', () => {
|
||||
const dbPath = path.join(tmpDir, 's3.db')
|
||||
fs.writeFileSync(dbPath, 'stable')
|
||||
assert.equal(getDbFileVersion(dbPath), getDbFileVersion(dbPath))
|
||||
})
|
||||
|
||||
it('does not throw when the file is missing', () => {
|
||||
assert.doesNotThrow(() => getDbFileVersion(path.join(tmpDir, 'missing.db')))
|
||||
})
|
||||
})
|
||||
|
||||
describe('getOrComputeAnalysisCache', () => {
|
||||
const KEY = 'wf:s1-e2'
|
||||
|
||||
it('computes once on cold miss and serves from cache on repeat (same version)', () => {
|
||||
let calls = 0
|
||||
const compute = () => {
|
||||
calls++
|
||||
return { words: ['a', 'b'], n: calls }
|
||||
}
|
||||
|
||||
const first = getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v1', compute)
|
||||
const second = getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v1', compute)
|
||||
|
||||
assert.equal(calls, 1, 'compute must run only once')
|
||||
assert.deepEqual(first, { words: ['a', 'b'], n: 1 })
|
||||
assert.deepEqual(second, first, 'second call must return the cached value')
|
||||
})
|
||||
|
||||
it('recomputes and overwrites when the version changes', () => {
|
||||
let calls = 0
|
||||
const compute = () => {
|
||||
calls++
|
||||
return { n: calls }
|
||||
}
|
||||
|
||||
getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v1', compute)
|
||||
const afterChange = getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v2', compute)
|
||||
|
||||
assert.equal(calls, 2, 'version change must trigger recompute')
|
||||
assert.deepEqual(afterChange, { n: 2 })
|
||||
|
||||
// 覆盖后,再用新版本读应命中、不再重算
|
||||
const cached = getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v2', compute)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(cached, { n: 2 })
|
||||
})
|
||||
|
||||
it('persists the result to {sessionId}.cache.json under the cache dir', () => {
|
||||
getOrComputeAnalysisCache('persist-sess', KEY, tmpDir, 'v1', () => ({ ok: true }))
|
||||
const file = path.join(tmpDir, 'persist-sess.cache.json')
|
||||
assert.ok(fs.existsSync(file), 'cache file must be written')
|
||||
const parsed = JSON.parse(fs.readFileSync(file, 'utf-8'))
|
||||
assert.equal(parsed[KEY].data.v, 'v1')
|
||||
assert.deepEqual(parsed[KEY].data.data, { ok: true })
|
||||
})
|
||||
|
||||
it('isolates entries by key within the same session file', () => {
|
||||
let calls = 0
|
||||
const compute = () => ({ n: ++calls })
|
||||
getOrComputeAnalysisCache('sess', 'k1', tmpDir, 'v1', compute)
|
||||
getOrComputeAnalysisCache('sess', 'k2', tmpDir, 'v1', compute)
|
||||
assert.equal(calls, 2, 'different keys must not collide')
|
||||
|
||||
getOrComputeAnalysisCache('sess', 'k1', tmpDir, 'v1', compute)
|
||||
assert.equal(calls, 2, 'existing key must still be a hit')
|
||||
})
|
||||
|
||||
it('falls back to recompute when the cache file is corrupted', () => {
|
||||
const file = path.join(tmpDir, 'sess.cache.json')
|
||||
fs.writeFileSync(file, '{ not valid json')
|
||||
|
||||
let calls = 0
|
||||
const result = getOrComputeAnalysisCache('sess', KEY, tmpDir, 'v1', () => ({ n: ++calls }))
|
||||
assert.equal(calls, 1)
|
||||
assert.deepEqual(result, { n: 1 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Analytics result cache (platform-agnostic).
|
||||
*
|
||||
* Caches expensive analytics / NLP computations (word frequency, catchphrase,
|
||||
* activity stats, ...) as JSON under {queryCacheDir}/{sessionId}.cache.json,
|
||||
* reusing the generic session-cache infrastructure.
|
||||
*
|
||||
* Each entry carries a `v` fingerprint derived from the session DB file state.
|
||||
* Any write to the chat DB (import, incremental import, member merge/rename,
|
||||
* owner change, summary generation) changes the file and thus the fingerprint,
|
||||
* forcing a recompute. This makes the cache self-validating even across
|
||||
* processes — e.g. when `chatlab start` is serving while a separate
|
||||
* `chatlab import` mutates the same database file.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs'
|
||||
import { getCache, setCache } from './session-cache'
|
||||
|
||||
interface VersionedEntry<T> {
|
||||
v: string
|
||||
data: T
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a cheap version fingerprint from the session DB file state.
|
||||
*
|
||||
* Combines mtime + size of the main DB file and its `-wal` sidecar so that
|
||||
* WAL-only writes (before a checkpoint) are detected too. Missing files
|
||||
* contribute a placeholder; a fully missing DB yields a stable placeholder
|
||||
* version, which is acceptable because the caller only computes once the DB
|
||||
* actually exists.
|
||||
*/
|
||||
export function getDbFileVersion(dbPath: string): string {
|
||||
const parts: string[] = []
|
||||
for (const p of [dbPath, `${dbPath}-wal`]) {
|
||||
try {
|
||||
const st = fs.statSync(p)
|
||||
parts.push(`${Math.floor(st.mtimeMs)}:${st.size}`)
|
||||
} catch {
|
||||
parts.push('-')
|
||||
}
|
||||
}
|
||||
return parts.join('|')
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache-first analytics read with version validation.
|
||||
*
|
||||
* Returns the cached value when the stored fingerprint matches `version`;
|
||||
* otherwise runs `compute()`, persists the result tagged with the current
|
||||
* version (overwriting any stale entry under the same key), and returns it.
|
||||
*/
|
||||
export function getOrComputeAnalysisCache<T>(
|
||||
sessionId: string,
|
||||
key: string,
|
||||
queryCacheDir: string,
|
||||
version: string,
|
||||
compute: () => T
|
||||
): T {
|
||||
const cached = getCache<VersionedEntry<T>>(sessionId, key, queryCacheDir)
|
||||
if (cached && cached.v === version) {
|
||||
return cached.data
|
||||
}
|
||||
const data = compute()
|
||||
setCache<VersionedEntry<T>>(sessionId, key, { v: version, data }, queryCacheDir)
|
||||
return data
|
||||
}
|
||||
+1
@@ -11,3 +11,4 @@ export {
|
||||
CACHE_KEY_MEMBERS,
|
||||
} from './session-cache'
|
||||
export type { OverviewCache, MembersCache, MemberStat } from './session-cache'
|
||||
export { getDbFileVersion, getOrComputeAnalysisCache } from './analytics-cache'
|
||||
@@ -285,6 +285,8 @@ export {
|
||||
computeAndSetOverviewCache,
|
||||
computeAndSetMembersCache,
|
||||
getValidatedOverviewCache,
|
||||
getDbFileVersion,
|
||||
getOrComputeAnalysisCache,
|
||||
CACHE_KEY_OVERVIEW,
|
||||
CACHE_KEY_MEMBERS,
|
||||
} from './cache'
|
||||
|
||||
Reference in new issue
Block a user