diff --git a/apps/desktop/main/worker/import/incrementalImport.ts b/apps/desktop/main/worker/import/incrementalImport.ts index 26d2e037..598b6cff 100644 --- a/apps/desktop/main/worker/import/incrementalImport.ts +++ b/apps/desktop/main/worker/import/incrementalImport.ts @@ -6,12 +6,14 @@ * DB open, and overview cache hook. */ +import * as path from 'path' import Database from 'better-sqlite3' import { BetterSqliteAdapter, analyzeIncrementalImport as sharedAnalyze, incrementalImport as sharedImport, computeAndSetOverviewCache, + deleteSessionCache, } from '@openchatlab/node-runtime' import type { IncrementalImportDeps, @@ -41,13 +43,18 @@ function buildDeps(requestId: string): IncrementalImportDeps { sendProgress(requestId, progress) }, postImportHook(_db, sessionId) { + const cacheDir = getCacheDir() try { const dbPath = getDbPath(sessionId) const rawDb = new Database(dbPath) - computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, getCacheDir()) + computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, cacheDir) rawDb.close() - } catch { - /* non-fatal */ + } catch (err) { + // Non-fatal: getValidatedOverviewCache will recompute on next read. + console.warn('[Worker] postImportHook: failed to refresh overview cache', err) + } + if (cacheDir) { + deleteSessionCache(sessionId, path.join(cacheDir, 'query')) } }, } diff --git a/apps/desktop/main/worker/import/streamImport.ts b/apps/desktop/main/worker/import/streamImport.ts index a2170515..8145feae 100644 --- a/apps/desktop/main/worker/import/streamImport.ts +++ b/apps/desktop/main/worker/import/streamImport.ts @@ -16,6 +16,7 @@ import { streamParseFileInfo as sharedStreamParseFileInfo, TEMP_DB_SCHEMA, computeAndSetOverviewCache, + deleteSessionCache, } from '@openchatlab/node-runtime' import type { StreamImportDeps, StreamImportResult, ImportLogger } from '@openchatlab/node-runtime' import { sendProgress, generateSessionId, getDbPath, createDatabaseWithoutIndexes } from './utils' @@ -89,13 +90,17 @@ function buildStreamImportDeps(requestId: string): StreamImportDeps { }, logger: buildElectronLogger(), postImportHook(_db, sessionId) { + const cacheDir = getCacheDir() try { const dbPath = getDbPath(sessionId) const rawDb = new Database(dbPath) - computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, getCacheDir()) + computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, cacheDir) rawDb.close() - } catch { - /* non-fatal */ + } catch (err) { + console.warn('[Worker] postImportHook: failed to refresh overview cache', err) + } + if (cacheDir) { + deleteSessionCache(sessionId, path.join(cacheDir, 'query')) } }, generateSessionId, diff --git a/apps/desktop/main/worker/query/sessions.ts b/apps/desktop/main/worker/query/sessions.ts index 9a1c534c..a9065bd1 100644 --- a/apps/desktop/main/worker/query/sessions.ts +++ b/apps/desktop/main/worker/query/sessions.ts @@ -14,11 +14,9 @@ import * as path from 'path' import { openDatabase, getDbDir, getDbPath, getCacheDir } from '../core' import { getCache, - computeAndSetOverviewCache, + getValidatedOverviewCache, computeAndSetMembersCache, - CACHE_KEY_OVERVIEW, CACHE_KEY_MEMBERS, - type OverviewCache, type MembersCache, } from '@openchatlab/node-runtime' import { @@ -50,18 +48,20 @@ function isChatSessionDb(db: Database.Database): boolean { } /** - * Resolve overview from cache (with compute-on-miss), or fall back to live SQL via core. + * Resolve overview with fingerprint-validated cache, falling back to live SQL. + * + * Uses getValidatedOverviewCache which checks MAX(message.id) so the cache is + * automatically recomputed whenever new messages are inserted (e.g. after + * incremental import), without relying on the import hook always succeeding. */ function resolveOverview(db: Database.Database, sessionId: string, cacheDir: string | null): SessionOverview { - let cached = cacheDir ? getCache(sessionId, CACHE_KEY_OVERVIEW, cacheDir) : null - if (!cached && cacheDir) { + if (cacheDir) { try { - cached = computeAndSetOverviewCache(new BetterSqliteAdapter(db), sessionId, cacheDir) + return getValidatedOverviewCache(new BetterSqliteAdapter(db), sessionId, cacheDir) } catch { // cache compute failure — fall through to live query } } - if (cached) return cached return getSessionOverview(asCoreDb(db)) } diff --git a/packages/node-runtime/src/cache/index.ts b/packages/node-runtime/src/cache/index.ts index 99952267..44c5e5ef 100644 --- a/packages/node-runtime/src/cache/index.ts +++ b/packages/node-runtime/src/cache/index.ts @@ -6,6 +6,7 @@ export { deleteSessionCache, computeAndSetOverviewCache, computeAndSetMembersCache, + getValidatedOverviewCache, CACHE_KEY_OVERVIEW, CACHE_KEY_MEMBERS, } from './session-cache' diff --git a/packages/node-runtime/src/cache/session-cache.test.ts b/packages/node-runtime/src/cache/session-cache.test.ts new file mode 100644 index 00000000..5fecbfd9 --- /dev/null +++ b/packages/node-runtime/src/cache/session-cache.test.ts @@ -0,0 +1,147 @@ +/** + * Regression tests for overview cache self-invalidation. + * + * Key scenario: after incremental import inserts new messages the cache + * fingerprint (MAX(message.id)) must mismatch, triggering a recompute so + * the AI system prompt always sees the latest lastMessageTs. + */ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import Database from 'better-sqlite3' +import { CHAT_DB_SCHEMA } from '@openchatlab/core' +import { BetterSqliteAdapter } from '../better-sqlite3-adapter' +import { + computeAndSetOverviewCache, + getCache, + setCache, + getValidatedOverviewCache, + CACHE_KEY_OVERVIEW, + type OverviewCache, +} from './session-cache' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-session-cache-')) +} + +/** + * Create an in-memory (or temp-file) chat DB with schema and one member so + * sender_id FK joins work. + */ +function makeTestDb(filePath?: string): Database.Database { + const db = new Database(filePath ?? ':memory:', { nativeBinding }) + db.exec(CHAT_DB_SCHEMA) + // Insert a system member and a regular member + db.prepare( + `INSERT INTO member (platform_id, account_name, group_nickname, avatar, roles) + VALUES (?, ?, ?, NULL, '[]')` + ).run('sys', '系统消息', null) + db.prepare( + `INSERT INTO member (platform_id, account_name, group_nickname, avatar, roles) + VALUES (?, ?, ?, NULL, '[]')` + ).run('u1', 'Alice', 'Alice') + return db +} + +function insertMessage(db: Database.Database, senderId: number, ts: number): void { + db.prepare('INSERT INTO message (sender_id, ts, type, content) VALUES (?, ?, 0, ?)').run(senderId, ts, `msg at ${ts}`) +} + +// ────────────────────────────────────────────────────────────────────────────── +// Test 1: core regression — cache becomes stale after incremental import; must recompute +// ────────────────────────────────────────────────────────────────────────────── +test('getValidatedOverviewCache recomputes when new messages are inserted after caching', () => { + const tmpDir = makeTempDir() + const dbPath = path.join(tmpDir, 'test.db') + const db = makeTestDb(dbPath) + const adapter = new BetterSqliteAdapter(db) + + // Pre-import: two messages, latest ts=1000 + insertMessage(db, 2, 500) // sender_id 2 = Alice (id=2 after sys=1) + insertMessage(db, 2, 1000) + + // Build initial cache (simulates what postImportHook does after first import) + computeAndSetOverviewCache(adapter, 'test-session', tmpDir) + + // Verify cache was written with correct values + const cached = getCache('test-session', CACHE_KEY_OVERVIEW, tmpDir) + assert.ok(cached, 'cache should exist after computeAndSetOverviewCache') + assert.equal(cached.lastMessageTs, 1000, 'cached lastMessageTs should be 1000') + + // Simulate incremental import: insert a newer message (ts=2000) + // This is what happens when the user imports new chat data + insertMessage(db, 2, 2000) + + // NOW call getValidatedOverviewCache — it must detect the fingerprint mismatch + // and recompute, returning the updated lastMessageTs=2000 + const result = getValidatedOverviewCache(adapter, 'test-session', tmpDir) + assert.equal( + result.lastMessageTs, + 2000, + 'getValidatedOverviewCache must return updated lastMessageTs=2000 after incremental import' + ) +}) + +// ────────────────────────────────────────────────────────────────────────────── +// Test 2: cache hit — no new messages means fingerprint matches, no recompute +// ────────────────────────────────────────────────────────────────────────────── +test('getValidatedOverviewCache returns cached value without recompute when data is unchanged', () => { + const tmpDir = makeTempDir() + const dbPath = path.join(tmpDir, 'test.db') + const db = makeTestDb(dbPath) + const adapter = new BetterSqliteAdapter(db) + + insertMessage(db, 2, 1000) + + computeAndSetOverviewCache(adapter, 'test-session', tmpDir) + + // Inject a stale lastMessageTs into the cache to prove we are NOT recomputing + const staleOverride: OverviewCache & { maxMessageId: number } = { + totalMessages: 1, + totalMembers: 1, + firstMessageTs: 1000, + lastMessageTs: 999, // intentionally wrong — we want to detect if this is returned as-is + maxMessageId: (db.prepare('SELECT MAX(id) AS m FROM message').get() as { m: number }).m, + } + setCache('test-session', CACHE_KEY_OVERVIEW, staleOverride, tmpDir) + + // Since no new messages were inserted, fingerprint still matches + // The function must return the cached value (999) without recomputing + const result = getValidatedOverviewCache(adapter, 'test-session', tmpDir) + assert.equal(result.lastMessageTs, 999, 'should return cached value unchanged when fingerprint matches (no new data)') +}) + +// ────────────────────────────────────────────────────────────────────────────── +// Test 3: legacy cache without maxMessageId triggers recompute +// ────────────────────────────────────────────────────────────────────────────── +test('getValidatedOverviewCache recomputes for old cache files that lack maxMessageId', () => { + const tmpDir = makeTempDir() + const dbPath = path.join(tmpDir, 'test.db') + const db = makeTestDb(dbPath) + const adapter = new BetterSqliteAdapter(db) + + insertMessage(db, 2, 1500) + + // Manually write a legacy-format cache entry WITHOUT maxMessageId + const legacyCache: OverviewCache = { + totalMessages: 0, + totalMembers: 0, + firstMessageTs: null, + lastMessageTs: null, + // maxMessageId intentionally absent (legacy format) + } + setCache('test-session', CACHE_KEY_OVERVIEW, legacyCache, tmpDir) + + // Must recompute because maxMessageId is missing from cached value + const result = getValidatedOverviewCache(adapter, 'test-session', tmpDir) + assert.equal( + result.lastMessageTs, + 1500, + 'should recompute and return real lastMessageTs when legacy cache lacks maxMessageId' + ) +}) diff --git a/packages/node-runtime/src/cache/session-cache.ts b/packages/node-runtime/src/cache/session-cache.ts index 14343793..de809b05 100644 --- a/packages/node-runtime/src/cache/session-cache.ts +++ b/packages/node-runtime/src/cache/session-cache.ts @@ -95,6 +95,23 @@ export interface OverviewCache { totalMembers: number firstMessageTs: number | null lastMessageTs: number | null + /** + * MAX(message.id) at the time the cache was written. + * Used as a cheap O(1) freshness fingerprint: if current MAX(id) differs + * the cache is stale and must be recomputed. Absent in legacy cache files + * written before this field was added; absence is treated as stale. + */ + maxMessageId?: number +} + +/** + * Return the current MAX(message.id) for freshness checking. + * This is O(1) via the AUTOINCREMENT rowid B-tree. + * Returns 0 for an empty table. + */ +function getMaxMessageId(db: DatabaseAdapter): number { + const row = db.prepare('SELECT MAX(id) AS m FROM message').get() as { m: number | null } + return row.m ?? 0 } export function computeAndSetOverviewCache(db: DatabaseAdapter, sessionId: string, cacheDir: string): OverviewCache { @@ -124,6 +141,7 @@ export function computeAndSetOverviewCache(db: DatabaseAdapter, sessionId: strin totalMembers, firstMessageTs: msgStats.first_ts, lastMessageTs: msgStats.last_ts, + maxMessageId: getMaxMessageId(db), } setCache(sessionId, CACHE_KEY_OVERVIEW, data, cacheDir) @@ -132,6 +150,25 @@ export function computeAndSetOverviewCache(db: DatabaseAdapter, sessionId: strin return data } +/** + * Cache-first overview read with fingerprint validation. + * + * Checks cached `maxMessageId` against the live `MAX(message.id)`. If they + * match the cache is fresh and returned as-is (one extra O(1) query). + * On any mismatch — new inserts, legacy cache lacking the field, or a cold + * miss — the cache is recomputed and written before returning. + * + * Use this instead of bare `getCache` whenever you need a guaranteed-fresh + * overview (e.g. AI system prompt data-snapshot construction). + */ +export function getValidatedOverviewCache(db: DatabaseAdapter, sessionId: string, cacheDir: string): OverviewCache { + const cached = getCache(sessionId, CACHE_KEY_OVERVIEW, cacheDir) + if (cached && cached.maxMessageId !== undefined && cached.maxMessageId === getMaxMessageId(db)) { + return cached + } + return computeAndSetOverviewCache(db, sessionId, cacheDir) +} + // ==================== Members cache (per-member stats) ==================== export const CACHE_KEY_MEMBERS = 'members' diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index cd882daf..fbd34fe5 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -284,6 +284,7 @@ export { deleteSessionCache, computeAndSetOverviewCache, computeAndSetMembersCache, + getValidatedOverviewCache, CACHE_KEY_OVERVIEW, CACHE_KEY_MEMBERS, } from './cache'