mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-05-14 18:39:07 +08:00
feat: 新增对话数据缓存管理模块,优化统计数据读取性能
This commit is contained in:
@@ -8,7 +8,8 @@ import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import type { ParseResult } from '../../../src/types/base'
|
||||
import { migrateDatabase, needsMigration, CURRENT_SCHEMA_VERSION } from './migrations'
|
||||
import { getDatabaseDir, ensureDir } from '../paths'
|
||||
import { getDatabaseDir, getCacheDir, ensureDir } from '../paths'
|
||||
import { deleteSessionCache } from './sessionCache'
|
||||
|
||||
/**
|
||||
* 获取数据库目录
|
||||
@@ -341,6 +342,7 @@ export function deleteSession(sessionId: string): boolean {
|
||||
if (fs.existsSync(shmPath)) {
|
||||
fs.unlinkSync(shmPath)
|
||||
}
|
||||
deleteSessionCache(sessionId, getCacheDir())
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error('[Database] Failed to delete session:', error)
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* 会话级通用 JSON 缓存模块
|
||||
*
|
||||
* 每个 session 对应一个 {sessionId}.cache.json,内部按 key 分区存储。
|
||||
* 与数据库 schema 完全解耦,读取失败自动重建,无需版本管理。
|
||||
*
|
||||
* 文件路径: {cacheDir}/{sessionId}.cache.json
|
||||
*/
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import type Database from 'better-sqlite3'
|
||||
|
||||
// ==================== 通用缓存基础设施 ====================
|
||||
|
||||
interface CacheEntry<T = unknown> {
|
||||
data: T
|
||||
ts: number
|
||||
}
|
||||
|
||||
type CacheFile = Record<string, CacheEntry>
|
||||
|
||||
export function getCachePath(sessionId: string, cacheDir: string): string {
|
||||
return path.join(cacheDir, `${sessionId}.cache.json`)
|
||||
}
|
||||
|
||||
function readCacheFile(cachePath: string): CacheFile | null {
|
||||
try {
|
||||
if (!fs.existsSync(cachePath)) return null
|
||||
return JSON.parse(fs.readFileSync(cachePath, 'utf-8')) as CacheFile
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function writeCacheFile(cachePath: string, content: CacheFile): void {
|
||||
try {
|
||||
const dir = path.dirname(cachePath)
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true })
|
||||
}
|
||||
fs.writeFileSync(cachePath, JSON.stringify(content), 'utf-8')
|
||||
} catch {
|
||||
// 写入失败不影响主流程
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取缓存分区
|
||||
*/
|
||||
export function getCache<T>(sessionId: string, key: string, cacheDir: string): T | null {
|
||||
const cachePath = getCachePath(sessionId, cacheDir)
|
||||
const file = readCacheFile(cachePath)
|
||||
if (!file || !file[key]) return null
|
||||
return file[key].data as T
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入缓存分区
|
||||
*/
|
||||
export function setCache<T>(sessionId: string, key: string, data: T, cacheDir: string): void {
|
||||
const cachePath = getCachePath(sessionId, cacheDir)
|
||||
const file = readCacheFile(cachePath) ?? {}
|
||||
file[key] = { data, ts: Math.floor(Date.now() / 1000) }
|
||||
writeCacheFile(cachePath, file)
|
||||
}
|
||||
|
||||
/**
|
||||
* 使缓存失效(不传 key 则清空整个文件)
|
||||
*/
|
||||
export function invalidateCache(sessionId: string, cacheDir: string, key?: string): void {
|
||||
const cachePath = getCachePath(sessionId, cacheDir)
|
||||
try {
|
||||
if (!key) {
|
||||
if (fs.existsSync(cachePath)) fs.unlinkSync(cachePath)
|
||||
} else {
|
||||
const file = readCacheFile(cachePath)
|
||||
if (file && file[key]) {
|
||||
delete file[key]
|
||||
writeCacheFile(cachePath, file)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 忽略失败
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除 session 的所有缓存
|
||||
*/
|
||||
export function deleteSessionCache(sessionId: string, cacheDir: string): void {
|
||||
const cachePath = getCachePath(sessionId, cacheDir)
|
||||
try {
|
||||
if (fs.existsSync(cachePath)) fs.unlinkSync(cachePath)
|
||||
} catch {
|
||||
// 忽略删除失败
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Overview 缓存(业务层) ====================
|
||||
|
||||
export const CACHE_KEY_OVERVIEW = 'overview'
|
||||
|
||||
export interface OverviewCache {
|
||||
totalMessages: number
|
||||
totalMembers: number
|
||||
firstMessageTs: number | null
|
||||
lastMessageTs: number | null
|
||||
/** member.id -> message count */
|
||||
memberMessageCounts: Record<number, number>
|
||||
}
|
||||
|
||||
/**
|
||||
* 从数据库计算概览统计并写入缓存
|
||||
*/
|
||||
export function computeAndSetOverviewCache(
|
||||
db: Database.Database,
|
||||
sessionId: string,
|
||||
cacheDir: string
|
||||
): OverviewCache {
|
||||
const msgStats = db.prepare('SELECT MIN(ts) as first_ts, MAX(ts) as last_ts FROM message').get() as {
|
||||
first_ts: number | null
|
||||
last_ts: number | null
|
||||
}
|
||||
|
||||
const totalMessages = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM message msg
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
WHERE COALESCE(m.account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
|
||||
const totalMembers = (
|
||||
db.prepare(`SELECT COUNT(*) as count FROM member WHERE COALESCE(account_name, '') != '系统消息'`).get() as {
|
||||
count: number
|
||||
}
|
||||
).count
|
||||
|
||||
const memberCounts = db
|
||||
.prepare('SELECT sender_id, COUNT(*) as count FROM message GROUP BY sender_id')
|
||||
.all() as Array<{ sender_id: number; count: number }>
|
||||
|
||||
const memberMessageCounts: Record<number, number> = {}
|
||||
for (const row of memberCounts) {
|
||||
memberMessageCounts[row.sender_id] = row.count
|
||||
}
|
||||
|
||||
const data: OverviewCache = {
|
||||
totalMessages,
|
||||
totalMembers,
|
||||
firstMessageTs: msgStats.first_ts,
|
||||
lastMessageTs: msgStats.last_ts,
|
||||
memberMessageCounts,
|
||||
}
|
||||
|
||||
setCache(sessionId, CACHE_KEY_OVERVIEW, data, cacheDir)
|
||||
return data
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import type { IpcContext } from './types'
|
||||
import {
|
||||
getAppDataDir,
|
||||
getDatabaseDir,
|
||||
getCacheDir,
|
||||
getAiDataDir,
|
||||
getLogsDir,
|
||||
getDownloadsDir,
|
||||
@@ -92,6 +93,14 @@ export function registerCacheHandlers(_context: IpcContext): void {
|
||||
icon: 'i-heroicons-sparkles',
|
||||
canClear: false, // 不允许一键清理
|
||||
},
|
||||
{
|
||||
id: 'cache',
|
||||
name: 'settings.storage.cache.statsCache.name',
|
||||
description: 'settings.storage.cache.statsCache.description',
|
||||
path: getCacheDir(),
|
||||
icon: 'i-heroicons-bolt',
|
||||
canClear: true,
|
||||
},
|
||||
// 临时文件已有自动清理机制(应用启动时、合并完成后),无需暴露给用户
|
||||
{
|
||||
id: 'logs',
|
||||
@@ -185,8 +194,8 @@ export function registerCacheHandlers(_context: IpcContext): void {
|
||||
* 清理指定缓存目录
|
||||
*/
|
||||
ipcMain.handle('cache:clear', async (_, cacheId: string) => {
|
||||
// 只允许清理 logs(temp 由系统自动清理,downloads 已改为系统下载目录)
|
||||
const allowedDirs: Record<string, string> = {
|
||||
cache: getCacheDir(),
|
||||
logs: getLogsDir(),
|
||||
}
|
||||
|
||||
@@ -271,6 +280,7 @@ export function registerCacheHandlers(_context: IpcContext): void {
|
||||
const dirPaths: Record<string, string> = {
|
||||
base: getAppDataDir(),
|
||||
databases: getDatabaseDir(),
|
||||
cache: getCacheDir(),
|
||||
ai: getAiDataDir(),
|
||||
logs: getLogsDir(),
|
||||
downloads: getDownloadsDir(), // 系统下载目录
|
||||
|
||||
@@ -368,6 +368,13 @@ export function getSettingsDir(): string {
|
||||
return path.join(getAppDataDir(), 'settings')
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取缓存目录(存放可再生的派生数据,如统计缓存)
|
||||
*/
|
||||
export function getCacheDir(): string {
|
||||
return path.join(getAppDataDir(), 'cache')
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取临时文件目录
|
||||
*/
|
||||
@@ -398,6 +405,7 @@ export function ensureAppDirs(): void {
|
||||
ensureDir(getDatabaseDir())
|
||||
ensureDir(getAiDataDir())
|
||||
ensureDir(getSettingsDir())
|
||||
ensureDir(getCacheDir())
|
||||
ensureDir(getTempDir())
|
||||
ensureDir(getLogsDir())
|
||||
// 写入数据目录标记文件
|
||||
|
||||
@@ -9,6 +9,8 @@ import * as path from 'path'
|
||||
|
||||
// 数据库目录(由 Worker 初始化时设置)
|
||||
let DB_DIR: string = ''
|
||||
// 缓存目录(由 Worker 初始化时设置)
|
||||
let CACHE_DIR: string = ''
|
||||
|
||||
// 数据库连接缓存
|
||||
const dbCache = new Map<string, Database.Database>()
|
||||
@@ -16,8 +18,9 @@ const dbCache = new Map<string, Database.Database>()
|
||||
/**
|
||||
* 初始化数据库目录
|
||||
*/
|
||||
export function initDbDir(dir: string): void {
|
||||
export function initDbDir(dir: string, cacheDir?: string): void {
|
||||
DB_DIR = dir
|
||||
if (cacheDir) CACHE_DIR = cacheDir
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -77,6 +80,10 @@ export function getDbDir(): string {
|
||||
return DB_DIR
|
||||
}
|
||||
|
||||
export function getCacheDir(): string {
|
||||
return CACHE_DIR
|
||||
}
|
||||
|
||||
// ==================== 时间过滤工具 ====================
|
||||
|
||||
export interface TimeFilter {
|
||||
|
||||
@@ -10,6 +10,7 @@ export {
|
||||
closeDatabase,
|
||||
closeAllDatabases,
|
||||
getDbDir,
|
||||
getCacheDir,
|
||||
buildTimeFilter,
|
||||
buildSystemMessageFilter,
|
||||
type TimeFilter,
|
||||
|
||||
@@ -68,7 +68,7 @@ import {
|
||||
import { streamImport, streamParseFileInfo, analyzeIncrementalImport, incrementalImport } from './import'
|
||||
|
||||
// 初始化数据库目录
|
||||
initDbDir(workerData.dbDir)
|
||||
initDbDir(workerData.dbDir, workerData.cacheDir)
|
||||
|
||||
// ==================== 消息处理 ====================
|
||||
|
||||
|
||||
@@ -243,6 +243,15 @@ export async function incrementalImport(
|
||||
// 更新 imported_at 时间
|
||||
db.prepare('UPDATE meta SET imported_at = ?').run(Math.floor(Date.now() / 1000))
|
||||
|
||||
// 写入概览统计缓存文件
|
||||
try {
|
||||
const { computeAndSetOverviewCache } = await import('../../database/sessionCache')
|
||||
const { getCacheDir } = await import('../core')
|
||||
computeAndSetOverviewCache(db, sessionId, getCacheDir())
|
||||
} catch {
|
||||
// 缓存写入失败不影响导入流程
|
||||
}
|
||||
|
||||
db.close()
|
||||
|
||||
sendProgress(requestId, {
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
type FormatFeature,
|
||||
} from '../../parser'
|
||||
import { ChatType } from '../../../../src/types/base'
|
||||
import { getDbDir } from '../core'
|
||||
import { getDbDir, getCacheDir } from '../core'
|
||||
import {
|
||||
initPerfLog,
|
||||
logPerf,
|
||||
@@ -714,6 +714,16 @@ async function streamImportSingle(
|
||||
})
|
||||
doCheckpoint()
|
||||
logPerf('WAL checkpoint done', totalMessageCount)
|
||||
|
||||
// 写入概览统计缓存文件
|
||||
try {
|
||||
const { computeAndSetOverviewCache } = await import('../../database/sessionCache')
|
||||
computeAndSetOverviewCache(db, sessionId, getCacheDir())
|
||||
logPerf('Stats cache written', totalMessageCount)
|
||||
} catch {
|
||||
// 缓存写入失败不影响导入流程
|
||||
}
|
||||
|
||||
logPerf('Import completed', totalMessageCount)
|
||||
|
||||
// 记录解析器回调统计(诊断信息)
|
||||
|
||||
@@ -9,10 +9,12 @@ import {
|
||||
openDatabase,
|
||||
closeDatabase,
|
||||
getDbPath,
|
||||
getCacheDir,
|
||||
buildTimeFilter,
|
||||
buildSystemMessageFilter,
|
||||
type TimeFilter,
|
||||
} from '../core'
|
||||
import { getCache, CACHE_KEY_OVERVIEW, type OverviewCache } from '../../database/sessionCache'
|
||||
|
||||
// ==================== 基础查询 ====================
|
||||
|
||||
@@ -383,15 +385,17 @@ export function getMessageLengthDistribution(
|
||||
* 获取时间范围
|
||||
*/
|
||||
export function getTimeRange(sessionId: string): { start: number; end: number } | null {
|
||||
// 优先从缓存读取
|
||||
const overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, getCacheDir())
|
||||
if (overview?.firstMessageTs != null && overview?.lastMessageTs != null) {
|
||||
return { start: overview.firstMessageTs, end: overview.lastMessageTs }
|
||||
}
|
||||
|
||||
const db = openDatabase(sessionId)
|
||||
if (!db) return null
|
||||
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT MIN(ts) as start, MAX(ts) as end FROM message
|
||||
`
|
||||
)
|
||||
.prepare('SELECT MIN(ts) as start, MAX(ts) as end FROM message')
|
||||
.get() as { start: number | null; end: number | null }
|
||||
|
||||
if (row.start === null || row.end === null) return null
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
import Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { openDatabase, getDbDir, getDbPath } from '../core'
|
||||
import { openDatabase, getDbDir, getDbPath, getCacheDir } from '../core'
|
||||
import { getCache, computeAndSetOverviewCache, CACHE_KEY_OVERVIEW, type OverviewCache } from '../../database/sessionCache'
|
||||
|
||||
interface DbMeta {
|
||||
name: string
|
||||
@@ -94,25 +95,40 @@ export function getAllSessions(): any[] {
|
||||
const meta = db.prepare('SELECT * FROM meta LIMIT 1').get() as DbMeta | undefined
|
||||
|
||||
if (meta) {
|
||||
const messageCount = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM message msg
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
WHERE COALESCE(m.account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
const memberCount = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM member
|
||||
WHERE COALESCE(account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
// 优先从缓存读取,未命中则实时查询并生成缓存
|
||||
const cacheDir = getCacheDir()
|
||||
let overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, cacheDir)
|
||||
if (!overview && cacheDir) {
|
||||
try {
|
||||
overview = computeAndSetOverviewCache(db, sessionId, cacheDir)
|
||||
} catch {
|
||||
// 缓存计算失败不影响正常流程
|
||||
}
|
||||
}
|
||||
|
||||
const messageCount =
|
||||
overview?.totalMessages ??
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM message msg
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
WHERE COALESCE(m.account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
const memberCount =
|
||||
overview?.totalMembers ??
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM member
|
||||
WHERE COALESCE(account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
|
||||
// 私聊:获取对方成员头像
|
||||
let memberAvatar: string | null = null
|
||||
@@ -174,26 +190,32 @@ export function getSession(sessionId: string): any | null {
|
||||
const meta = db.prepare('SELECT * FROM meta LIMIT 1').get() as DbMeta | undefined
|
||||
if (!meta) return null
|
||||
|
||||
const messageCount = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
const overview = getCache<OverviewCache>(sessionId, CACHE_KEY_OVERVIEW, getCacheDir())
|
||||
|
||||
const messageCount =
|
||||
overview?.totalMessages ??
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM message msg
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
WHERE COALESCE(m.account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
|
||||
const memberCount = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
const memberCount =
|
||||
overview?.totalMembers ??
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM member
|
||||
WHERE COALESCE(account_name, '') != '系统消息'`
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count
|
||||
|
||||
return {
|
||||
id: sessionId,
|
||||
|
||||
@@ -9,7 +9,7 @@ import * as path from 'path'
|
||||
import type { ParseProgress } from '../parser'
|
||||
import type { StreamImportResult } from './import'
|
||||
import { openDatabase } from '../database/core'
|
||||
import { getDatabaseDir, ensureDir } from '../paths'
|
||||
import { getDatabaseDir, getCacheDir, ensureDir } from '../paths'
|
||||
|
||||
// Worker 实例
|
||||
let worker: Worker | null = null
|
||||
@@ -69,6 +69,7 @@ export function initWorker(): void {
|
||||
worker = new Worker(workerPath, {
|
||||
workerData: {
|
||||
dbDir: getDbDir(),
|
||||
cacheDir: getCacheDir(),
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -157,6 +157,10 @@
|
||||
"name": "Downloads",
|
||||
"description": "Screenshots, analysis results, etc."
|
||||
},
|
||||
"statsCache": {
|
||||
"name": "Cache Data",
|
||||
"description": "Statistics cache for chat records, auto-rebuilt after clearing"
|
||||
},
|
||||
"logs": {
|
||||
"name": "Log Files",
|
||||
"description": "App logs including import, AI, error logs"
|
||||
|
||||
@@ -157,6 +157,10 @@
|
||||
"name": "ダウンロードディレクトリ",
|
||||
"description": "スクリーンショットや分析結果などを含む"
|
||||
},
|
||||
"statsCache": {
|
||||
"name": "キャッシュデータ",
|
||||
"description": "チャット記録の統計キャッシュ。クリア後に自動で再構築されます"
|
||||
},
|
||||
"logs": {
|
||||
"name": "ログファイル",
|
||||
"description": "アプリの実行ログ(インポート、AI、エラーなどのログを含む)"
|
||||
|
||||
@@ -157,6 +157,10 @@
|
||||
"name": "下载目录",
|
||||
"description": "包含截屏文件、分析结果等"
|
||||
},
|
||||
"statsCache": {
|
||||
"name": "缓存数据",
|
||||
"description": "聊天记录的统计缓存,清理后会自动重建"
|
||||
},
|
||||
"logs": {
|
||||
"name": "日志文件",
|
||||
"description": "软件的运行日志,包含导入、AI、错误等日志"
|
||||
|
||||
@@ -157,6 +157,10 @@
|
||||
"name": "下載目錄",
|
||||
"description": "包含截圖檔案、分析結果等"
|
||||
},
|
||||
"statsCache": {
|
||||
"name": "快取資料",
|
||||
"description": "聊天記錄的統計快取,清理後會自動重建"
|
||||
},
|
||||
"logs": {
|
||||
"name": "日誌檔案",
|
||||
"description": "軟體的執行日誌,包含匯入、AI、錯誤等日誌"
|
||||
|
||||
@@ -231,6 +231,7 @@ defineExpose({
|
||||
:class="{
|
||||
'bg-green-100 dark:bg-green-900/30': dir.id === 'databases',
|
||||
'bg-violet-100 dark:bg-violet-900/30': dir.id === 'ai',
|
||||
'bg-cyan-100 dark:bg-cyan-900/30': dir.id === 'cache',
|
||||
'bg-amber-100 dark:bg-amber-900/30': dir.id === 'downloads',
|
||||
'bg-blue-100 dark:bg-blue-900/30': dir.id === 'logs',
|
||||
}"
|
||||
@@ -241,6 +242,7 @@ defineExpose({
|
||||
:class="{
|
||||
'text-green-600 dark:text-green-400': dir.id === 'databases',
|
||||
'text-violet-600 dark:text-violet-400': dir.id === 'ai',
|
||||
'text-cyan-600 dark:text-cyan-400': dir.id === 'cache',
|
||||
'text-amber-600 dark:text-amber-400': dir.id === 'downloads',
|
||||
'text-blue-600 dark:text-blue-400': dir.id === 'logs',
|
||||
}"
|
||||
|
||||
Reference in New Issue
Block a user