feat: 新增对话数据缓存管理模块,优化统计数据读取性能

This commit is contained in:
digua
2026-03-29 11:55:15 +08:00
committed by digua
parent bfca7ff133
commit 211a8110b0
17 changed files with 298 additions and 45 deletions
+3 -1
View File
@@ -8,7 +8,8 @@ import * as fs from 'fs'
import * as path from 'path'
import type { ParseResult } from '../../../src/types/base'
import { migrateDatabase, needsMigration, CURRENT_SCHEMA_VERSION } from './migrations'
import { getDatabaseDir, ensureDir } from '../paths'
import { getDatabaseDir, getCacheDir, ensureDir } from '../paths'
import { deleteSessionCache } from './sessionCache'
/**
* 获取数据库目录
@@ -341,6 +342,7 @@ export function deleteSession(sessionId: string): boolean {
if (fs.existsSync(shmPath)) {
fs.unlinkSync(shmPath)
}
deleteSessionCache(sessionId, getCacheDir())
return true
} catch (error) {
console.error('[Database] Failed to delete session:', error)
+161
View File
@@ -0,0 +1,161 @@
/**
* 会话级通用 JSON 缓存模块
*
* 每个 session 对应一个 {sessionId}.cache.json,内部按 key 分区存储。
* 与数据库 schema 完全解耦,读取失败自动重建,无需版本管理。
*
* 文件路径: {cacheDir}/{sessionId}.cache.json
*/
import * as fs from 'fs'
import * as path from 'path'
import type Database from 'better-sqlite3'
// ==================== 通用缓存基础设施 ====================
interface CacheEntry<T = unknown> {
data: T
ts: number
}
type CacheFile = Record<string, CacheEntry>
export function getCachePath(sessionId: string, cacheDir: string): string {
return path.join(cacheDir, `${sessionId}.cache.json`)
}
function readCacheFile(cachePath: string): CacheFile | null {
try {
if (!fs.existsSync(cachePath)) return null
return JSON.parse(fs.readFileSync(cachePath, 'utf-8')) as CacheFile
} catch {
return null
}
}
function writeCacheFile(cachePath: string, content: CacheFile): void {
try {
const dir = path.dirname(cachePath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
fs.writeFileSync(cachePath, JSON.stringify(content), 'utf-8')
} catch {
// 写入失败不影响主流程
}
}
/**
* 读取缓存分区
*/
export function getCache<T>(sessionId: string, key: string, cacheDir: string): T | null {
const cachePath = getCachePath(sessionId, cacheDir)
const file = readCacheFile(cachePath)
if (!file || !file[key]) return null
return file[key].data as T
}
/**
* 写入缓存分区
*/
export function setCache<T>(sessionId: string, key: string, data: T, cacheDir: string): void {
const cachePath = getCachePath(sessionId, cacheDir)
const file = readCacheFile(cachePath) ?? {}
file[key] = { data, ts: Math.floor(Date.now() / 1000) }
writeCacheFile(cachePath, file)
}
/**
* 使缓存失效(不传 key 则清空整个文件)
*/
export function invalidateCache(sessionId: string, cacheDir: string, key?: string): void {
const cachePath = getCachePath(sessionId, cacheDir)
try {
if (!key) {
if (fs.existsSync(cachePath)) fs.unlinkSync(cachePath)
} else {
const file = readCacheFile(cachePath)
if (file && file[key]) {
delete file[key]
writeCacheFile(cachePath, file)
}
}
} catch {
// 忽略失败
}
}
/**
* 删除 session 的所有缓存
*/
export function deleteSessionCache(sessionId: string, cacheDir: string): void {
const cachePath = getCachePath(sessionId, cacheDir)
try {
if (fs.existsSync(cachePath)) fs.unlinkSync(cachePath)
} catch {
// 忽略删除失败
}
}
// ==================== Overview 缓存(业务层) ====================
export const CACHE_KEY_OVERVIEW = 'overview'
export interface OverviewCache {
totalMessages: number
totalMembers: number
firstMessageTs: number | null
lastMessageTs: number | null
/** member.id -> message count */
memberMessageCounts: Record<number, number>
}
/**
* 从数据库计算概览统计并写入缓存
*/
export function computeAndSetOverviewCache(
db: Database.Database,
sessionId: string,
cacheDir: string
): OverviewCache {
const msgStats = db.prepare('SELECT MIN(ts) as first_ts, MAX(ts) as last_ts FROM message').get() as {
first_ts: number | null
last_ts: number | null
}
const totalMessages = (
db
.prepare(
`SELECT COUNT(*) as count FROM message msg
JOIN member m ON msg.sender_id = m.id
WHERE COALESCE(m.account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
const totalMembers = (
db.prepare(`SELECT COUNT(*) as count FROM member WHERE COALESCE(account_name, '') != '系统消息'`).get() as {
count: number
}
).count
const memberCounts = db
.prepare('SELECT sender_id, COUNT(*) as count FROM message GROUP BY sender_id')
.all() as Array<{ sender_id: number; count: number }>
const memberMessageCounts: Record<number, number> = {}
for (const row of memberCounts) {
memberMessageCounts[row.sender_id] = row.count
}
const data: OverviewCache = {
totalMessages,
totalMembers,
firstMessageTs: msgStats.first_ts,
lastMessageTs: msgStats.last_ts,
memberMessageCounts,
}
setCache(sessionId, CACHE_KEY_OVERVIEW, data, cacheDir)
return data
}
+11 -1
View File
@@ -7,6 +7,7 @@ import type { IpcContext } from './types'
import {
getAppDataDir,
getDatabaseDir,
getCacheDir,
getAiDataDir,
getLogsDir,
getDownloadsDir,
@@ -92,6 +93,14 @@ export function registerCacheHandlers(_context: IpcContext): void {
icon: 'i-heroicons-sparkles',
canClear: false, // 不允许一键清理
},
{
id: 'cache',
name: 'settings.storage.cache.statsCache.name',
description: 'settings.storage.cache.statsCache.description',
path: getCacheDir(),
icon: 'i-heroicons-bolt',
canClear: true,
},
// 临时文件已有自动清理机制(应用启动时、合并完成后),无需暴露给用户
{
id: 'logs',
@@ -185,8 +194,8 @@ export function registerCacheHandlers(_context: IpcContext): void {
* 清理指定缓存目录
*/
ipcMain.handle('cache:clear', async (_, cacheId: string) => {
// 只允许清理 logstemp 由系统自动清理,downloads 已改为系统下载目录)
const allowedDirs: Record<string, string> = {
cache: getCacheDir(),
logs: getLogsDir(),
}
@@ -271,6 +280,7 @@ export function registerCacheHandlers(_context: IpcContext): void {
const dirPaths: Record<string, string> = {
base: getAppDataDir(),
databases: getDatabaseDir(),
cache: getCacheDir(),
ai: getAiDataDir(),
logs: getLogsDir(),
downloads: getDownloadsDir(), // 系统下载目录
+8
View File
@@ -368,6 +368,13 @@ export function getSettingsDir(): string {
return path.join(getAppDataDir(), 'settings')
}
/**
* 获取缓存目录(存放可再生的派生数据,如统计缓存)
*/
export function getCacheDir(): string {
return path.join(getAppDataDir(), 'cache')
}
/**
* 获取临时文件目录
*/
@@ -398,6 +405,7 @@ export function ensureAppDirs(): void {
ensureDir(getDatabaseDir())
ensureDir(getAiDataDir())
ensureDir(getSettingsDir())
ensureDir(getCacheDir())
ensureDir(getTempDir())
ensureDir(getLogsDir())
// 写入数据目录标记文件
+8 -1
View File
@@ -9,6 +9,8 @@ import * as path from 'path'
// 数据库目录(由 Worker 初始化时设置)
let DB_DIR: string = ''
// 缓存目录(由 Worker 初始化时设置)
let CACHE_DIR: string = ''
// 数据库连接缓存
const dbCache = new Map<string, Database.Database>()
@@ -16,8 +18,9 @@ const dbCache = new Map<string, Database.Database>()
/**
* 初始化数据库目录
*/
export function initDbDir(dir: string): void {
export function initDbDir(dir: string, cacheDir?: string): void {
DB_DIR = dir
if (cacheDir) CACHE_DIR = cacheDir
}
/**
@@ -77,6 +80,10 @@ export function getDbDir(): string {
return DB_DIR
}
export function getCacheDir(): string {
return CACHE_DIR
}
// ==================== 时间过滤工具 ====================
export interface TimeFilter {
+1
View File
@@ -10,6 +10,7 @@ export {
closeDatabase,
closeAllDatabases,
getDbDir,
getCacheDir,
buildTimeFilter,
buildSystemMessageFilter,
type TimeFilter,
+1 -1
View File
@@ -68,7 +68,7 @@ import {
import { streamImport, streamParseFileInfo, analyzeIncrementalImport, incrementalImport } from './import'
// 初始化数据库目录
initDbDir(workerData.dbDir)
initDbDir(workerData.dbDir, workerData.cacheDir)
// ==================== 消息处理 ====================
@@ -243,6 +243,15 @@ export async function incrementalImport(
// 更新 imported_at 时间
db.prepare('UPDATE meta SET imported_at = ?').run(Math.floor(Date.now() / 1000))
// 写入概览统计缓存文件
try {
const { computeAndSetOverviewCache } = await import('../../database/sessionCache')
const { getCacheDir } = await import('../core')
computeAndSetOverviewCache(db, sessionId, getCacheDir())
} catch {
// 缓存写入失败不影响导入流程
}
db.close()
sendProgress(requestId, {
+11 -1
View File
@@ -18,7 +18,7 @@ import {
type FormatFeature,
} from '../../parser'
import { ChatType } from '../../../../src/types/base'
import { getDbDir } from '../core'
import { getDbDir, getCacheDir } from '../core'
import {
initPerfLog,
logPerf,
@@ -714,6 +714,16 @@ async function streamImportSingle(
})
doCheckpoint()
logPerf('WAL checkpoint done', totalMessageCount)
// 写入概览统计缓存文件
try {
const { computeAndSetOverviewCache } = await import('../../database/sessionCache')
computeAndSetOverviewCache(db, sessionId, getCacheDir())
logPerf('Stats cache written', totalMessageCount)
} catch {
// 缓存写入失败不影响导入流程
}
logPerf('Import completed', totalMessageCount)
// 记录解析器回调统计(诊断信息)
+9 -5
View File
@@ -9,10 +9,12 @@ import {
openDatabase,
closeDatabase,
getDbPath,
getCacheDir,
buildTimeFilter,
buildSystemMessageFilter,
type TimeFilter,
} from '../core'
import { getCache, CACHE_KEY_OVERVIEW, type OverviewCache } from '../../database/sessionCache'
// ==================== 基础查询 ====================
@@ -383,15 +385,17 @@ export function getMessageLengthDistribution(
* 获取时间范围
*/
export function getTimeRange(sessionId: string): { start: number; end: number } | null {
// 优先从缓存读取
const overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, getCacheDir())
if (overview?.firstMessageTs != null && overview?.lastMessageTs != null) {
return { start: overview.firstMessageTs, end: overview.lastMessageTs }
}
const db = openDatabase(sessionId)
if (!db) return null
const row = db
.prepare(
`
SELECT MIN(ts) as start, MAX(ts) as end FROM message
`
)
.prepare('SELECT MIN(ts) as start, MAX(ts) as end FROM message')
.get() as { start: number | null; end: number | null }
if (row.start === null || row.end === null) return null
+56 -34
View File
@@ -6,7 +6,8 @@
import Database from 'better-sqlite3'
import * as fs from 'fs'
import * as path from 'path'
import { openDatabase, getDbDir, getDbPath } from '../core'
import { openDatabase, getDbDir, getDbPath, getCacheDir } from '../core'
import { getCache, computeAndSetOverviewCache, CACHE_KEY_OVERVIEW, type OverviewCache } from '../../database/sessionCache'
interface DbMeta {
name: string
@@ -94,25 +95,40 @@ export function getAllSessions(): any[] {
const meta = db.prepare('SELECT * FROM meta LIMIT 1').get() as DbMeta | undefined
if (meta) {
const messageCount = (
db
.prepare(
`SELECT COUNT(*) as count
FROM message msg
JOIN member m ON msg.sender_id = m.id
WHERE COALESCE(m.account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
const memberCount = (
db
.prepare(
`SELECT COUNT(*) as count
FROM member
WHERE COALESCE(account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
// 优先从缓存读取,未命中则实时查询并生成缓存
const cacheDir = getCacheDir()
let overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, cacheDir)
if (!overview && cacheDir) {
try {
overview = computeAndSetOverviewCache(db, sessionId, cacheDir)
} catch {
// 缓存计算失败不影响正常流程
}
}
const messageCount =
overview?.totalMessages ??
(
db
.prepare(
`SELECT COUNT(*) as count
FROM message msg
JOIN member m ON msg.sender_id = m.id
WHERE COALESCE(m.account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
const memberCount =
overview?.totalMembers ??
(
db
.prepare(
`SELECT COUNT(*) as count
FROM member
WHERE COALESCE(account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
// 私聊:获取对方成员头像
let memberAvatar: string | null = null
@@ -174,26 +190,32 @@ export function getSession(sessionId: string): any | null {
const meta = db.prepare('SELECT * FROM meta LIMIT 1').get() as DbMeta | undefined
if (!meta) return null
const messageCount = (
db
.prepare(
`SELECT COUNT(*) as count
const overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, getCacheDir())
const messageCount =
overview?.totalMessages ??
(
db
.prepare(
`SELECT COUNT(*) as count
FROM message msg
JOIN member m ON msg.sender_id = m.id
WHERE COALESCE(m.account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
)
.get() as { count: number }
).count
const memberCount = (
db
.prepare(
`SELECT COUNT(*) as count
const memberCount =
overview?.totalMembers ??
(
db
.prepare(
`SELECT COUNT(*) as count
FROM member
WHERE COALESCE(account_name, '') != '系统消息'`
)
.get() as { count: number }
).count
)
.get() as { count: number }
).count
return {
id: sessionId,
+2 -1
View File
@@ -9,7 +9,7 @@ import * as path from 'path'
import type { ParseProgress } from '../parser'
import type { StreamImportResult } from './import'
import { openDatabase } from '../database/core'
import { getDatabaseDir, ensureDir } from '../paths'
import { getDatabaseDir, getCacheDir, ensureDir } from '../paths'
// Worker 实例
let worker: Worker | null = null
@@ -69,6 +69,7 @@ export function initWorker(): void {
worker = new Worker(workerPath, {
workerData: {
dbDir: getDbDir(),
cacheDir: getCacheDir(),
},
})
+4
View File
@@ -157,6 +157,10 @@
"name": "Downloads",
"description": "Screenshots, analysis results, etc."
},
"statsCache": {
"name": "Cache Data",
"description": "Statistics cache for chat records, auto-rebuilt after clearing"
},
"logs": {
"name": "Log Files",
"description": "App logs including import, AI, error logs"
+4
View File
@@ -157,6 +157,10 @@
"name": "ダウンロードディレクトリ",
"description": "スクリーンショットや分析結果などを含む"
},
"statsCache": {
"name": "キャッシュデータ",
"description": "チャット記録の統計キャッシュ。クリア後に自動で再構築されます"
},
"logs": {
"name": "ログファイル",
"description": "アプリの実行ログ(インポート、AI、エラーなどのログを含む)"
+4
View File
@@ -157,6 +157,10 @@
"name": "下载目录",
"description": "包含截屏文件、分析结果等"
},
"statsCache": {
"name": "缓存数据",
"description": "聊天记录的统计缓存,清理后会自动重建"
},
"logs": {
"name": "日志文件",
"description": "软件的运行日志,包含导入、AI、错误等日志"
+4
View File
@@ -157,6 +157,10 @@
"name": "下載目錄",
"description": "包含截圖檔案、分析結果等"
},
"statsCache": {
"name": "快取資料",
"description": "聊天記錄的統計快取,清理後會自動重建"
},
"logs": {
"name": "日誌檔案",
"description": "軟體的執行日誌,包含匯入、AI、錯誤等日誌"
@@ -231,6 +231,7 @@ defineExpose({
:class="{
'bg-green-100 dark:bg-green-900/30': dir.id === 'databases',
'bg-violet-100 dark:bg-violet-900/30': dir.id === 'ai',
'bg-cyan-100 dark:bg-cyan-900/30': dir.id === 'cache',
'bg-amber-100 dark:bg-amber-900/30': dir.id === 'downloads',
'bg-blue-100 dark:bg-blue-900/30': dir.id === 'logs',
}"
@@ -241,6 +242,7 @@ defineExpose({
:class="{
'text-green-600 dark:text-green-400': dir.id === 'databases',
'text-violet-600 dark:text-violet-400': dir.id === 'ai',
'text-cyan-600 dark:text-cyan-400': dir.id === 'cache',
'text-amber-600 dark:text-amber-400': dir.id === 'downloads',
'text-blue-600 dark:text-blue-400': dir.id === 'logs',
}"