From a5bbff34e45d9e10fce0df0840dba78f7072a4aa Mon Sep 17 00:00:00 2001 From: digua Date: Fri, 22 May 2026 21:46:08 +0800 Subject: [PATCH] feat(sync): use incremental session index after pull sync After pull-sync imports new messages, generate session index incrementally instead of full regeneration. This preserves existing sessions and their summaries, only processing newly imported (unindexed) messages. Adds generateIncremental method across the full stack: - core: already had generateIncrementalSessionIndex - node-runtime: expose generateIncrementalIndex - CLI: add POST /_web/sessions/:id/generate-incremental-index route - Electron: add session:generateIncremental IPC handler - Frontend: add generateIncremental to SessionIndexAdapter interface --- apps/cli/src/http/routes/web/session-index.ts | 9 +++++++++ apps/desktop/main/ipc/chat.ts | 9 +++++++++ apps/desktop/preload/apis/utils.ts | 7 ++++--- apps/desktop/preload/index.d.ts | 1 + packages/node-runtime/src/services/index.ts | 9 ++++++++- .../src/services/session-index-service.ts | 10 ++++++++++ src/services/session-index/electron.ts | 4 ++++ src/services/session-index/fetch.ts | 11 +++++++++++ src/services/session-index/types.ts | 1 + src/stores/apiServer.ts | 4 ++-- 10 files changed, 59 insertions(+), 6 deletions(-) diff --git a/apps/cli/src/http/routes/web/session-index.ts b/apps/cli/src/http/routes/web/session-index.ts index e6bc6ce5..16a03693 100644 --- a/apps/cli/src/http/routes/web/session-index.ts +++ b/apps/cli/src/http/routes/web/session-index.ts @@ -11,6 +11,15 @@ export function registerSessionIndexRoutes(server: FastifyInstance, adapter: Ses return { sessionCount } }) + server.post<{ + Params: { id: string } + Body: { gapThreshold?: number } + }>('/_web/sessions/:id/generate-incremental-index', async (request) => { + const gapThreshold = (request.body as any)?.gapThreshold ?? 1800 + const sessionCount = sessionIndexService.generateIncrementalIndex(adapter, request.params.id, gapThreshold) + return { sessionCount } + }) + server.post<{ Params: { id: string } }>('/_web/sessions/:id/clear-index', async (request) => { sessionIndexService.clearIndex(adapter, request.params.id) return { success: true } diff --git a/apps/desktop/main/ipc/chat.ts b/apps/desktop/main/ipc/chat.ts index 415d3b72..ce6b8ffe 100644 --- a/apps/desktop/main/ipc/chat.ts +++ b/apps/desktop/main/ipc/chat.ts @@ -803,6 +803,15 @@ export function registerChatHandlers(ctx: IpcContext): void { } }) + ipcMain.handle('session:generateIncremental', async (_, sessionId: string, gapThreshold?: number) => { + try { + return await worker.generateIncrementalSessions(sessionId, gapThreshold) + } catch (error) { + console.error('Failed to generate incremental session index:', error) + throw error + } + }) + /** * 检查是否已生成会话索引 */ diff --git a/apps/desktop/preload/apis/utils.ts b/apps/desktop/preload/apis/utils.ts index 3437faa2..4932e7de 100644 --- a/apps/desktop/preload/apis/utils.ts +++ b/apps/desktop/preload/apis/utils.ts @@ -240,9 +240,10 @@ export const sessionApi = { return ipcRenderer.invoke('session:generate', sessionId, gapThreshold) }, - /** - * 检查是否已生成会话索引 - */ + generateIncremental: (sessionId: string, gapThreshold?: number): Promise => { + return ipcRenderer.invoke('session:generateIncremental', sessionId, gapThreshold) + }, + hasIndex: (sessionId: string): Promise => { return ipcRenderer.invoke('session:hasIndex', sessionId) }, diff --git a/apps/desktop/preload/index.d.ts b/apps/desktop/preload/index.d.ts index 628e0c2f..c319da40 100644 --- a/apps/desktop/preload/index.d.ts +++ b/apps/desktop/preload/index.d.ts @@ -1107,6 +1107,7 @@ interface ChatSessionItem { interface SessionApi { generate: (sessionId: string, gapThreshold?: number) => Promise + generateIncremental: (sessionId: string, gapThreshold?: number) => Promise hasIndex: (sessionId: string) => Promise getStats: (sessionId: string) => Promise clear: (sessionId: string) => Promise diff --git a/packages/node-runtime/src/services/index.ts b/packages/node-runtime/src/services/index.ts index 992d04c5..a037c95a 100644 --- a/packages/node-runtime/src/services/index.ts +++ b/packages/node-runtime/src/services/index.ts @@ -24,7 +24,14 @@ export { export type { MembersPaginatedDTO } from './member-service' // Session index service -export { generateIndex, clearIndex, getFtsStatus, searchFts, rebuildFts } from './session-index-service' +export { + generateIndex, + generateIncrementalIndex, + clearIndex, + getFtsStatus, + searchFts, + rebuildFts, +} from './session-index-service' // Summary service export { generateSummary, generateAllSummaries } from './summary-service' diff --git a/packages/node-runtime/src/services/session-index-service.ts b/packages/node-runtime/src/services/session-index-service.ts index 96f45791..9b2adcd3 100644 --- a/packages/node-runtime/src/services/session-index-service.ts +++ b/packages/node-runtime/src/services/session-index-service.ts @@ -6,6 +6,7 @@ import { generateSessionIndex as coreGenerateSessionIndex, + generateIncrementalSessionIndex as coreGenerateIncrementalSessionIndex, clearSessionIndex as coreClearSessionIndex, } from '@openchatlab/core' import { hasFtsTable, searchByFts, rebuildFtsIndex } from '../fts' @@ -16,6 +17,15 @@ export function generateIndex(adapter: SessionRuntimeAdapter, sessionId: string, return coreGenerateSessionIndex(db, gapThreshold) } +export function generateIncrementalIndex( + adapter: SessionRuntimeAdapter, + sessionId: string, + gapThreshold: number = 1800 +): number { + const db = adapter.ensureWritable(sessionId) + return coreGenerateIncrementalSessionIndex(db, gapThreshold) +} + export function clearIndex(adapter: SessionRuntimeAdapter, sessionId: string): void { const db = adapter.ensureWritable(sessionId) coreClearSessionIndex(db) diff --git a/src/services/session-index/electron.ts b/src/services/session-index/electron.ts index 6bfc967d..c8c94c14 100644 --- a/src/services/session-index/electron.ts +++ b/src/services/session-index/electron.ts @@ -16,6 +16,10 @@ export class ElectronSessionIndexAdapter implements SessionIndexAdapter { return window.sessionApi.generate(sessionId, gapThreshold) } + generateIncremental(sessionId: string, gapThreshold?: number): Promise { + return window.sessionApi.generateIncremental(sessionId, gapThreshold) + } + hasIndex(sessionId: string): Promise { return window.sessionApi.hasIndex(sessionId) } diff --git a/src/services/session-index/fetch.ts b/src/services/session-index/fetch.ts index b3efeb94..2791dce4 100644 --- a/src/services/session-index/fetch.ts +++ b/src/services/session-index/fetch.ts @@ -33,6 +33,17 @@ export class FetchSessionIndexAdapter implements SessionIndexAdapter { return result.sessionCount } + async generateIncremental(sessionId: string, gapThreshold: number = 1800): Promise { + const resp = await fetch(`/_web/sessions/${sessionId}/generate-incremental-index`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ gapThreshold }), + }) + if (!resp.ok) throw new Error(`Failed to generate incremental session index: ${resp.status}`) + const result = (await resp.json()) as { sessionCount: number } + return result.sessionCount + } + async hasIndex(sessionId: string): Promise { const stats = await this.getStats(sessionId) return stats.hasIndex diff --git a/src/services/session-index/types.ts b/src/services/session-index/types.ts index c6a246d4..51135fa5 100644 --- a/src/services/session-index/types.ts +++ b/src/services/session-index/types.ts @@ -39,6 +39,7 @@ export interface CanGenerateInfo { export interface SessionIndexAdapter { generate(sessionId: string, gapThreshold?: number): Promise + generateIncremental(sessionId: string, gapThreshold?: number): Promise hasIndex(sessionId: string): Promise getStats(sessionId: string): Promise clear(sessionId: string): Promise diff --git a/src/stores/apiServer.ts b/src/stores/apiServer.ts index 71e808b4..ebefd36e 100644 --- a/src/stores/apiServer.ts +++ b/src/stores/apiServer.ts @@ -526,7 +526,7 @@ export const useApiServerStore = defineStore('apiServer', () => { const indexService = useSessionIndexService() const threshold = getSessionGapThreshold() for (const sess of targets) { - indexService.generate(sess.targetSessionId, threshold).catch(() => {}) + indexService.generateIncremental(sess.targetSessionId, threshold).catch(() => {}) } } @@ -536,7 +536,7 @@ export const useApiServerStore = defineStore('apiServer', () => { for (const ds of dataSources.value) { for (const sess of ds.sessions) { if (sess.targetSessionId) { - indexService.generate(sess.targetSessionId, threshold).catch(() => {}) + indexService.generateIncremental(sess.targetSessionId, threshold).catch(() => {}) } } }