From ced4ed16bdea68267ee74dd11fee8925e3e0c09c Mon Sep 17 00:00:00 2001 From: digua Date: Thu, 25 Jun 2026 10:43:33 +0800 Subject: [PATCH] feat(import): auto-generate session index after all import paths Session index (segment/message_context tables) was previously generated inconsistently: CLI used postImportHook, Desktop relied on a separate explicit RPC call, and the CLI sync/pull legacy importer never built it. Move generateSessionIndex / generateIncrementalSessionIndex into the shared importers (streaming-importer.ts, incremental-importer.ts) so every caller gets it automatically. Remove the now-redundant calls from CLI's postImportHook. Also add the calls to the legacy importer.ts used by CLI sync/pull, which had no session index at all. Incremental imports use generateIncrementalSessionIndex, which preserves existing AI summaries. Full imports call generateSessionIndex only on newly created sessions where no summaries exist. Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/import/importer.ts | 20 ++++++++++++++++++- apps/cli/src/import/stream-import.ts | 15 -------------- .../src/import/incremental-importer.ts | 11 +++++++++- .../src/import/streaming-importer.ts | 10 +++++++++- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/apps/cli/src/import/importer.ts b/apps/cli/src/import/importer.ts index 5c9b1fd4..298308ac 100644 --- a/apps/cli/src/import/importer.ts +++ b/apps/cli/src/import/importer.ts @@ -8,7 +8,14 @@ import * as fs from 'fs' import * as crypto from 'crypto' import type { DatabaseAdapter } from '@openchatlab/core' -import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey, buildMemberIdMap } from '@openchatlab/core' +import { + CHAT_DB_SCHEMA, + FTS_TABLE_SCHEMA, + generateMessageKey, + buildMemberIdMap, + generateSessionIndex, + generateIncrementalSessionIndex, +} from '@openchatlab/core' import type { DatabaseManager } from '@openchatlab/node-runtime' import { writeParseResultToDb } from '@openchatlab/node-runtime' import type { ParsedData, ImportMessage } from './chatlab-reader' @@ -51,6 +58,12 @@ function fullImport( buildFts(db, onProgress) + try { + generateSessionIndex(db) + } catch { + /* non-fatal */ + } + return { messageCount: stats.messageCount, memberCount: stats.memberCount, duplicateCount: stats.skippedCount } } @@ -161,6 +174,11 @@ function incrementalImport( if (written > 0) { insertFtsEntries(db, onProgress) db.prepare('UPDATE meta SET imported_at = ?').run(Math.floor(Date.now() / 1000)) + try { + generateIncrementalSessionIndex(db) + } catch { + /* non-fatal */ + } } return { messageCount: written, memberCount: newMembers, duplicateCount: duplicates } diff --git a/apps/cli/src/import/stream-import.ts b/apps/cli/src/import/stream-import.ts index 5a572f66..909673f0 100644 --- a/apps/cli/src/import/stream-import.ts +++ b/apps/cli/src/import/stream-import.ts @@ -14,7 +14,6 @@ import { analyzeIncrementalImport as sharedAnalyzeIncremental, incrementalImport as sharedIncrementalImport, } from '@openchatlab/node-runtime' -import { generateSessionIndex, generateIncrementalSessionIndex } from '@openchatlab/core' import type { StreamImportResult, StreamImportDeps, @@ -71,13 +70,6 @@ function buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportPr dbManager.deleteSessionDatabaseFiles(sessionId) }, onProgress: onProgress ?? (() => {}), - postImportHook(db) { - try { - generateSessionIndex(db) - } catch { - /* non-fatal: frontend can regenerate via generate-index route */ - } - }, generateSessionId, } } @@ -174,13 +166,6 @@ function buildIncrementalDeps( } }, onProgress: onProgress ?? (() => {}), - postImportHook(db) { - try { - generateIncrementalSessionIndex(db) - } catch { - /* non-fatal */ - } - }, } } diff --git a/packages/node-runtime/src/import/incremental-importer.ts b/packages/node-runtime/src/import/incremental-importer.ts index 12ebaa86..9c55e846 100644 --- a/packages/node-runtime/src/import/incremental-importer.ts +++ b/packages/node-runtime/src/import/incremental-importer.ts @@ -9,7 +9,7 @@ */ import type { DatabaseAdapter } from '@openchatlab/core' -import { generateMessageKey } from '@openchatlab/core' +import { generateMessageKey, generateIncrementalSessionIndex } from '@openchatlab/core' import { streamParseFile, detectFormat, type ParseProgress } from '@openchatlab/parser' import { insertFtsEntries, hasFtsTable } from '../fts' import type { ImportProgressCallback } from './streaming-importer' @@ -379,6 +379,15 @@ export async function incrementalImport( } } + // Incremental session index (segment / message_context tables) + if (newMessageCount > 0) { + try { + generateIncrementalSessionIndex(db) + } catch { + /* non-fatal */ + } + } + const sessionStats = db .prepare( `SELECT diff --git a/packages/node-runtime/src/import/streaming-importer.ts b/packages/node-runtime/src/import/streaming-importer.ts index 499bd053..350bd48a 100644 --- a/packages/node-runtime/src/import/streaming-importer.ts +++ b/packages/node-runtime/src/import/streaming-importer.ts @@ -10,7 +10,7 @@ */ import type { DatabaseAdapter } from '@openchatlab/core' -import { CHAT_DB_INDEXES } from '@openchatlab/core' +import { CHAT_DB_INDEXES, generateSessionIndex } from '@openchatlab/core' import type { ParsedMember, ParsedMessage } from '@openchatlab/shared-types' import { streamParseFile, @@ -518,6 +518,14 @@ async function streamImportSingle( doCheckpoint() logger?.perf('WAL checkpoint done', totalMessageCount) + // Build session index (segment / message_context tables) + try { + generateSessionIndex(db) + logger?.perf('Session index built', totalMessageCount) + } catch { + /* non-fatal */ + } + // Post-import hook (e.g. overview cache) try { await deps.postImportHook?.(db, sessionId)