From 064f3ce144a966b6873de5ab59cc5f6bf81acce8 Mon Sep 17 00:00:00 2001 From: digua Date: Mon, 18 May 2026 20:38:38 +0800 Subject: [PATCH] feat(node-runtime): extract chat DB migrations and add auto-migration on writable open - Extract migration definitions (v1-v4) to shared getChatDbMigrations() with DI for tokenizer - DatabaseManager.openWritable() auto-runs pending migrations - Electron migrations.ts becomes thin wrapper delegating to shared definitions --- electron/main/database/migrations.ts | 194 ++---------------- packages/node-runtime/src/database-manager.ts | 44 +++- packages/node-runtime/src/index.ts | 4 + .../src/migrations/chat-db-migrations.ts | 152 ++++++++++++++ packages/node-runtime/src/migrations/index.ts | 2 + 5 files changed, 218 insertions(+), 178 deletions(-) create mode 100644 packages/node-runtime/src/migrations/chat-db-migrations.ts create mode 100644 packages/node-runtime/src/migrations/index.ts diff --git a/electron/main/database/migrations.ts b/electron/main/database/migrations.ts index 07f44b12..6df0ebfe 100644 --- a/electron/main/database/migrations.ts +++ b/electron/main/database/migrations.ts @@ -1,191 +1,39 @@ /** - * 数据库迁移系统 + * Database migration — Electron adapter layer. * - * 迁移脚本定义在 Electron 层（依赖 NLP 分词器、i18n 等平台特性）， - * 但执行引擎委托给 @openchatlab/core 的 runMigrations。 + * Thin wrapper around @openchatlab/node-runtime shared migration definitions. + * Keeps Electron-only concerns: i18n MigrationInfo, better-sqlite3 type bridging. */ import type Database from 'better-sqlite3' -import { - CURRENT_SCHEMA_VERSION, - runMigrations, - needsMigration as coreNeedsMigration, - type Migration as CoreMigration, -} from '@openchatlab/core' +import { CURRENT_SCHEMA_VERSION, runMigrations, needsMigration as coreNeedsMigration } from '@openchatlab/core' import type { DatabaseAdapter } from '@openchatlab/core' -import { BetterSqliteAdapter } from '@openchatlab/node-runtime' +import { BetterSqliteAdapter, getChatDbMigrations } from '@openchatlab/node-runtime' import { t } from '../i18n' import { tokenizeForFts } from '../nlp/ftsTokenizer' export { CURRENT_SCHEMA_VERSION } -/** 导出给前端使用的迁移信息 */ export interface MigrationInfo { version: number - /** 技术描述（面向开发者） */ description: string - /** 用户可读的升级原因（显示在弹窗中） */ userMessage: string } -interface ElectronMigration extends CoreMigration { - descriptionKey: string - userMessageKey: string -} - -const migrations: ElectronMigration[] = [ - { - version: 1, - description: 'Add owner_id column to meta', - descriptionKey: 'database.migrationV1Desc', - userMessageKey: 'database.migrationV1Message', - up: (db: DatabaseAdapter) => { - const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }> - const hasOwnerIdColumn = tableInfo.some((col) => col.name === 'owner_id') - if (!hasOwnerIdColumn) { - db.exec('ALTER TABLE meta ADD COLUMN owner_id TEXT') - } - }, - }, - { - version: 2, - description: 'Add roles, reply_to_message_id, platform_message_id columns', - descriptionKey: 'database.migrationV2Desc', - userMessageKey: 'database.migrationV2Message', - up: (db: DatabaseAdapter) => { - const memberTableInfo = db.pragma('table_info(member)') as Array<{ name: string }> - const hasRolesColumn = memberTableInfo.some((col) => col.name === 'roles') - if (!hasRolesColumn) { - db.exec("ALTER TABLE member ADD COLUMN roles TEXT DEFAULT '[]'") - } - - const messageTableInfo = db.pragma('table_info(message)') as Array<{ name: string }> - - const hasReplyColumn = messageTableInfo.some((col) => col.name === 'reply_to_message_id') - if (!hasReplyColumn) { - db.exec('ALTER TABLE message ADD COLUMN reply_to_message_id TEXT DEFAULT NULL') - } - - const hasPlatformMsgIdColumn = messageTableInfo.some((col) => col.name === 'platform_message_id') - if (!hasPlatformMsgIdColumn) { - db.exec('ALTER TABLE message ADD COLUMN platform_message_id TEXT DEFAULT NULL') - } - - try { - db.exec('CREATE INDEX IF NOT EXISTS idx_message_platform_id ON message(platform_message_id)') - } catch { - // Index may already exist - } - }, - }, - { - version: 3, - description: 'Add chat_session and message_context tables', - descriptionKey: 'database.migrationV3Desc', - userMessageKey: 'database.migrationV3Message', - up: (db: DatabaseAdapter) => { - db.exec(` - CREATE TABLE IF NOT EXISTS chat_session ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - start_ts INTEGER NOT NULL, - end_ts INTEGER NOT NULL, - message_count INTEGER DEFAULT 0, - is_manual INTEGER DEFAULT 0, - summary TEXT - ) - `) - - try { - db.exec('CREATE INDEX IF NOT EXISTS idx_session_time ON chat_session(start_ts, end_ts)') - } catch { - // Index may already exist - } - - db.exec(` - CREATE TABLE IF NOT EXISTS message_context ( - message_id INTEGER PRIMARY KEY, - session_id INTEGER NOT NULL, - topic_id INTEGER - ) - `) - - try { - db.exec('CREATE INDEX IF NOT EXISTS idx_context_session ON message_context(session_id)') - } catch { - // Index may already exist - } - - const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }> - const hasGapThresholdColumn = tableInfo.some((col) => col.name === 'session_gap_threshold') - if (!hasGapThresholdColumn) { - db.exec('ALTER TABLE meta ADD COLUMN session_gap_threshold INTEGER') - } - }, - }, - { - version: 4, - description: 'Add FTS5 full-text search index', - descriptionKey: 'database.migrationV4Desc', - userMessageKey: 'database.migrationV4Message', - up: (db: DatabaseAdapter) => { - const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get() - if (hasTable) return - - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( - content, - content='', - content_rowid=id - ) - `) - - const BATCH_SIZE = 5000 - const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)') - - const countRow = db - .prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''") - .get() as { total: number } | undefined - - const total = countRow?.total ?? 0 - let offset = 0 - while (offset < total) { - const rows = db - .prepare( - `SELECT id, content FROM message - WHERE type = 0 AND content IS NOT NULL AND content != '' - ORDER BY id ASC LIMIT ? OFFSET ?` - ) - .all(BATCH_SIZE, offset) as Array<{ id: number; content: string }> - - if (rows.length === 0) break - - for (const row of rows) { - const tokens = tokenizeForFts(row.content) - if (tokens) { - insertFts.run(row.id, tokens) - } - } - - offset += BATCH_SIZE - } - }, - }, +const i18nKeys: Array<{ descriptionKey: string; userMessageKey: string }> = [ + { descriptionKey: 'database.migrationV1Desc', userMessageKey: 'database.migrationV1Message' }, + { descriptionKey: 'database.migrationV2Desc', userMessageKey: 'database.migrationV2Message' }, + { descriptionKey: 'database.migrationV3Desc', userMessageKey: 'database.migrationV3Message' }, + { descriptionKey: 'database.migrationV4Desc', userMessageKey: 'database.migrationV4Message' }, ] -/** - * Check database structural integrity (meta table must exist) - */ function checkDatabaseIntegrity(db: DatabaseAdapter): { valid: boolean; error?: string } { try { const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'").all() as Array<{ name: string }> - if (tables.length === 0) { - return { - valid: false, - error: t('database.integrityError'), - } + return { valid: false, error: t('database.integrityError') } } return { valid: true } } catch (error) { @@ -198,10 +46,8 @@ function checkDatabaseIntegrity(db: DatabaseAdapter): { valid: boolean; error?: /** * Execute database migrations. - * Wraps better-sqlite3 Database in a DatabaseAdapter and delegates to core runMigrations. - * - * @param db Raw better-sqlite3 connection (stays open after migration) - * @param forceRepair Re-run all migration scripts even if version is current + * Wraps better-sqlite3 Database in a DatabaseAdapter and delegates to core runMigrations + * with shared migration definitions from @openchatlab/node-runtime. */ export function migrateDatabase(db: Database.Database, forceRepair = false): boolean { const adapter = new BetterSqliteAdapter(db) @@ -211,27 +57,25 @@ export function migrateDatabase(db: Database.Database, forceRepair = false): boo throw new Error(integrity.error) } + const migrations = getChatDbMigrations({ tokenizeForFts }) return runMigrations(adapter, migrations, forceRepair) } -/** - * Check if database needs migration - */ export function needsMigration(db: Database.Database): boolean { const adapter = new BetterSqliteAdapter(db) return coreNeedsMigration(adapter, CURRENT_SCHEMA_VERSION) } /** - * Get pending migration info for UI display - * @param fromVersion Starting version (exclusive) + * Get pending migration info for UI display (with i18n). */ export function getPendingMigrationInfos(fromVersion = 0): MigrationInfo[] { + const migrations = getChatDbMigrations({ tokenizeForFts }) return migrations .filter((m) => m.version > fromVersion) - .map((m) => ({ + .map((m, idx) => ({ version: m.version, - description: t(m.descriptionKey), - userMessage: t(m.userMessageKey), + description: i18nKeys[idx] ? t(i18nKeys[idx].descriptionKey) : m.description, + userMessage: i18nKeys[idx] ? t(i18nKeys[idx].userMessageKey) : m.description, })) } diff --git a/packages/node-runtime/src/database-manager.ts b/packages/node-runtime/src/database-manager.ts index 821f7e2e..2dff68df 100644 --- a/packages/node-runtime/src/database-manager.ts +++ b/packages/node-runtime/src/database-manager.ts @@ -9,22 +9,30 @@ import * as fs from 'fs' import * as path from 'path' import type { DatabaseAdapter, PathProvider } from '@openchatlab/core' -import { isChatSessionDb } from '@openchatlab/core' +import { + isChatSessionDb, + runMigrations, + needsMigration as coreNeedsMigration, + CURRENT_SCHEMA_VERSION, +} from '@openchatlab/core' import { openBetterSqliteDatabase } from './better-sqlite3-adapter' +import { getChatDbMigrations, type MigrationDeps } from './migrations' export class DatabaseManager { private cache = new Map() private nativeBinding?: string + private migrationDeps?: MigrationDeps constructor( private pathProvider: PathProvider, - options?: { nativeBinding?: string } + options?: { nativeBinding?: string; migrationDeps?: MigrationDeps } ) { this.nativeBinding = options?.nativeBinding + this.migrationDeps = options?.migrationDeps } /** - * 打开指定会话的数据库（带缓存） + * Open a session DB (read-only by default, cached). */ open(sessionId: string, options?: { readonly?: boolean }): DatabaseAdapter | null { if (this.cache.has(sessionId)) { @@ -42,6 +50,36 @@ export class DatabaseManager { return adapter } + /** + * Open a session DB in read-write mode and auto-run pending migrations. + * Falls back to `open(id, { readonly: false })` if migration is unnecessary. + */ + openWritable(sessionId: string): DatabaseAdapter | null { + const existing = this.cache.get(sessionId) + if (existing && !existing.readonly) return existing + + if (existing) { + existing.close() + this.cache.delete(sessionId) + } + + const dbPath = this.getDbPath(sessionId) + if (!fs.existsSync(dbPath)) return null + + const adapter = openBetterSqliteDatabase(dbPath, { + readonly: false, + nativeBinding: this.nativeBinding, + }) + + if (coreNeedsMigration(adapter, CURRENT_SCHEMA_VERSION)) { + const migrations = getChatDbMigrations(this.migrationDeps) + runMigrations(adapter, migrations) + } + + this.cache.set(sessionId, adapter) + return adapter + } + /** * 关闭指定会话的数据库连接 */ diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index 554e16c6..3263fd59 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -188,6 +188,10 @@ export type { PiModelConfig, BuildPiModelOptions } from './ai' export { fetchRemoteModels, validateApiKey } from './ai' export type { RemoteModel, FetchRemoteModelsResult, RemoteApiOptions } from './ai' +// Chat DB migrations +export { getChatDbMigrations } from './migrations' +export type { MigrationDeps } from './migrations' + // Merger orchestration export { checkConflictsFromSources, buildMergedOutput, serializeChatLabToJsonl } from './merger' export type { diff --git a/packages/node-runtime/src/migrations/chat-db-migrations.ts b/packages/node-runtime/src/migrations/chat-db-migrations.ts new file mode 100644 index 00000000..a5226c20 --- /dev/null +++ b/packages/node-runtime/src/migrations/chat-db-migrations.ts @@ -0,0 +1,152 @@ +/** + * Chat session DB migration definitions (platform-agnostic). + * + * Extracted from electron/main/database/migrations.ts. + * Migration scripts use only DatabaseAdapter — no Electron or Node-specific APIs. + * Version 4 (FTS backfill) requires a tokenizer injected via MigrationDeps. + */ + +import type { DatabaseAdapter } from '@openchatlab/core' +import type { Migration as CoreMigration } from '@openchatlab/core' + +export interface MigrationDeps { + /** FTS tokenizer — needed by v4 migration for backfilling the FTS index */ + tokenizeForFts?: (content: string) => string | null +} + +/** + * Build the chat DB migration list. + * + * @param deps Optional dependencies (tokenizer for FTS backfill) + * @returns Array of migrations compatible with core `runMigrations` + */ +export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] { + return [ + { + version: 1, + description: 'Add owner_id column to meta', + up: (db: DatabaseAdapter) => { + const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }> + if (!tableInfo.some((col) => col.name === 'owner_id')) { + db.exec('ALTER TABLE meta ADD COLUMN owner_id TEXT') + } + }, + }, + { + version: 2, + description: 'Add roles, reply_to_message_id, platform_message_id columns', + up: (db: DatabaseAdapter) => { + const memberTableInfo = db.pragma('table_info(member)') as Array<{ name: string }> + if (!memberTableInfo.some((col) => col.name === 'roles')) { + db.exec("ALTER TABLE member ADD COLUMN roles TEXT DEFAULT '[]'") + } + + const messageTableInfo = db.pragma('table_info(message)') as Array<{ name: string }> + + if (!messageTableInfo.some((col) => col.name === 'reply_to_message_id')) { + db.exec('ALTER TABLE message ADD COLUMN reply_to_message_id TEXT DEFAULT NULL') + } + + if (!messageTableInfo.some((col) => col.name === 'platform_message_id')) { + db.exec('ALTER TABLE message ADD COLUMN platform_message_id TEXT DEFAULT NULL') + } + + try { + db.exec('CREATE INDEX IF NOT EXISTS idx_message_platform_id ON message(platform_message_id)') + } catch { + // Index may already exist + } + }, + }, + { + version: 3, + description: 'Add chat_session and message_context tables', + up: (db: DatabaseAdapter) => { + db.exec(` + CREATE TABLE IF NOT EXISTS chat_session ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + start_ts INTEGER NOT NULL, + end_ts INTEGER NOT NULL, + message_count INTEGER DEFAULT 0, + is_manual INTEGER DEFAULT 0, + summary TEXT + ) + `) + + try { + db.exec('CREATE INDEX IF NOT EXISTS idx_session_time ON chat_session(start_ts, end_ts)') + } catch { + // Index may already exist + } + + db.exec(` + CREATE TABLE IF NOT EXISTS message_context ( + message_id INTEGER PRIMARY KEY, + session_id INTEGER NOT NULL, + topic_id INTEGER + ) + `) + + try { + db.exec('CREATE INDEX IF NOT EXISTS idx_context_session ON message_context(session_id)') + } catch { + // Index may already exist + } + + const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }> + if (!tableInfo.some((col) => col.name === 'session_gap_threshold')) { + db.exec('ALTER TABLE meta ADD COLUMN session_gap_threshold INTEGER') + } + }, + }, + { + version: 4, + description: 'Add FTS5 full-text search index', + up: (db: DatabaseAdapter) => { + const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get() + if (hasTable) return + + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( + content, + content='', + content_rowid=id + ) + `) + + const tokenize = deps?.tokenizeForFts + if (!tokenize) return + + const BATCH_SIZE = 5000 + const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)') + + const countRow = db + .prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''") + .get() as { total: number } | undefined + + const total = countRow?.total ?? 0 + let offset = 0 + while (offset < total) { + const rows = db + .prepare( + `SELECT id, content FROM message + WHERE type = 0 AND content IS NOT NULL AND content != '' + ORDER BY id ASC LIMIT ? OFFSET ?` + ) + .all(BATCH_SIZE, offset) as Array<{ id: number; content: string }> + + if (rows.length === 0) break + + for (const row of rows) { + const tokens = tokenize(row.content) + if (tokens) { + insertFts.run(row.id, tokens) + } + } + + offset += BATCH_SIZE + } + }, + }, + ] +} diff --git a/packages/node-runtime/src/migrations/index.ts b/packages/node-runtime/src/migrations/index.ts new file mode 100644 index 00000000..5d4e832a --- /dev/null +++ b/packages/node-runtime/src/migrations/index.ts @@ -0,0 +1,2 @@ +export { getChatDbMigrations } from './chat-db-migrations' +export type { MigrationDeps } from './chat-db-migrations'