From e7561e224e8c1c95e7b68fa630e7fe712795afde Mon Sep 17 00:00:00 2001 From: digua Date: Sun, 7 Jun 2026 22:36:44 +0800 Subject: [PATCH] fix: write data directory compatibility gate on migration and import Harden the startup migration path: fail fast and show an error dialog if schema migration cannot complete. Repair the gate retroactively for databases that were already migrated before the gate system existed. Raise the gate after every import path (full import, incremental import, stream import) and after opening any database that already carries a current-segment schema. Centralise gate-raising inside DatabaseManager so call sites cannot accidentally skip it. Co-Authored-By: Claude Sonnet 4.6 --- apps/cli/src/import/importer.test.ts | 102 +++++++++++++ apps/cli/src/import/importer.ts | 42 ++---- apps/cli/src/import/stream-import.test.ts | 81 ++++++++++ apps/cli/src/import/stream-import.ts | 88 +++++++---- apps/cli/src/sync/adapters.test.ts | 58 ++++++++ apps/cli/src/sync/adapters.ts | 20 +-- apps/desktop/main/api/adapters.ts | 6 +- apps/desktop/main/database/migrations.test.ts | 4 +- .../main/database/startup-migration.test.ts | 125 ++++++++++++++++ .../main/database/startup-migration.ts | 103 +++++++++++++ apps/desktop/main/index.ts | 35 +++-- apps/desktop/main/internal-api.ts | 28 ++-- apps/desktop/main/worker/workerManager.ts | 47 +++++- packages/mcp-server/src/bin.ts | 3 +- packages/mcp-server/src/env.d.ts | 1 + .../mcp-server/src/runtime-version.test.ts | 18 +++ packages/mcp-server/src/runtime-version.ts | 8 + packages/mcp-server/tsup.config.ts | 6 + .../node-runtime/src/data-dir-compat.test.ts | 23 +++ packages/node-runtime/src/data-dir-compat.ts | 15 +- .../node-runtime/src/database-manager.test.ts | 139 ++++++++++++++++++ packages/node-runtime/src/database-manager.ts | 96 ++++++++++-- packages/node-runtime/src/index.ts | 1 + .../src/migrations/chat-db-migrations.ts | 14 ++ packages/node-runtime/src/migrations/index.ts | 2 +- 25 files changed, 941 insertions(+), 124 deletions(-) create mode 100644 apps/cli/src/import/importer.test.ts create mode 100644 apps/cli/src/import/stream-import.test.ts create mode 100644 apps/cli/src/sync/adapters.test.ts create mode 100644 apps/desktop/main/database/startup-migration.test.ts create mode 100644 apps/desktop/main/database/startup-migration.ts create mode 100644 packages/mcp-server/src/env.d.ts create mode 100644 packages/mcp-server/src/runtime-version.test.ts create mode 100644 packages/mcp-server/src/runtime-version.ts diff --git a/apps/cli/src/import/importer.test.ts b/apps/cli/src/import/importer.test.ts new file mode 100644 index 00000000..d2dbcce8 --- /dev/null +++ b/apps/cli/src/import/importer.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, raiseDataDirMinRuntimeVersion, readDataDirCompatibilityMeta } from '@openchatlab/node-runtime' +import { importData } from './importer' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-import-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('importData raises the data directory gate after creating a current-schema database', async () => { + const root = makeTempDir() + fs.mkdirSync(path.join(root, 'data', 'databases'), { recursive: true }) + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const result = await importData( + manager, + { + meta: { name: 'Fresh Import', platform: 'qq', type: 'group' }, + members: [{ platformId: 'u1', accountName: 'Alice' }], + messages: [ + { + senderPlatformId: 'u1', + senderAccountName: 'Alice', + timestamp: 1000, + type: 0, + content: 'hello', + }, + ], + }, + { nativeBinding } + ) + + assert.equal(result.success, true) + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) + +test('importData re-checks data directory compatibility before raw database writes', async () => { + const root = makeTempDir() + fs.mkdirSync(path.join(root, 'data', 'databases'), { recursive: true }) + const pathProvider = createPathProvider(root) + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: '0.26.0', + dataCompatibilityVersion: 2, + reason: 'future-schema', + runtime: { version: '0.26.0', kind: 'desktop' }, + module: 'future-migration', + now: () => 1780830000, + }) + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const result = await importData( + manager, + { + meta: { name: 'Blocked Import', platform: 'qq', type: 'group' }, + members: [{ platformId: 'u1', accountName: 'Alice' }], + messages: [ + { + senderPlatformId: 'u1', + senderAccountName: 'Alice', + timestamp: 1000, + type: 0, + content: 'hello', + }, + ], + }, + { nativeBinding } + ) + + assert.equal(result.success, false) + assert.match(result.error ?? '', /requires runtime version 0\.26\.0 or newer/) + assert.equal(fs.readdirSync(path.join(root, 'data', 'databases')).filter((name) => name.endsWith('.db')).length, 0) +}) diff --git a/apps/cli/src/import/importer.ts b/apps/cli/src/import/importer.ts index ff4ceb14..5c9b1fd4 100644 --- a/apps/cli/src/import/importer.ts +++ b/apps/cli/src/import/importer.ts @@ -10,7 +10,7 @@ import * as crypto from 'crypto' import type { DatabaseAdapter } from '@openchatlab/core' import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey, buildMemberIdMap } from '@openchatlab/core' import type { DatabaseManager } from '@openchatlab/node-runtime' -import { openBetterSqliteDatabase, writeParseResultToDb } from '@openchatlab/node-runtime' +import { writeParseResultToDb } from '@openchatlab/node-runtime' import type { ParsedData, ImportMessage } from './chatlab-reader' export interface ImportResult { @@ -272,40 +272,30 @@ export async function importData( try { if (exists) { - dbManager.close(sessionId) - const db = openBetterSqliteDatabase(dbPath, { readonly: false, nativeBinding: options?.nativeBinding }) + const db = dbManager.openRawSessionDatabase(sessionId, { readonly: false }) + let result: { messageCount: number; memberCount: number; duplicateCount: number } try { - const result = incrementalImport(db, data, options?.onProgress) + result = incrementalImport(db, data, options?.onProgress) + } finally { db.close() - return { success: true, sessionId, created: false, ...result } - } catch (err) { - db.close() - throw err } + dbManager.raiseCurrentChatDbCompatibilityGate() + return { success: true, sessionId, created: false, ...result } } - const db = openBetterSqliteDatabase(dbPath, { nativeBinding: options?.nativeBinding }) - const result = fullImport(db, data, options?.onProgress) - db.close() + const db = dbManager.openRawSessionDatabase(sessionId, { create: true }) + let result: { messageCount: number; memberCount: number; duplicateCount: number } + try { + result = fullImport(db, data, options?.onProgress) + } finally { + db.close() + } + dbManager.raiseCurrentChatDbCompatibilityGate() return { success: true, sessionId, created: true, ...result } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err) if (!exists) { - try { - fs.unlinkSync(dbPath) - } catch { - /* ignore */ - } - try { - fs.unlinkSync(dbPath + '-wal') - } catch { - /* ignore */ - } - try { - fs.unlinkSync(dbPath + '-shm') - } catch { - /* ignore */ - } + dbManager.deleteSessionDatabaseFiles(sessionId) } return { success: false, diff --git a/apps/cli/src/import/stream-import.test.ts b/apps/cli/src/import/stream-import.test.ts new file mode 100644 index 00000000..e5beb8a5 --- /dev/null +++ b/apps/cli/src/import/stream-import.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, readDataDirCompatibilityMeta } from '@openchatlab/node-runtime' +import { incrementalImport } from './stream-import' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-stream-import-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +function writeIncrementalJsonl(filePath: string): void { + const rows = [ + { + _type: 'header', + chatlab: { version: '1.0.0', exportedAt: 1780830000 }, + meta: { name: 'Incremental Chat', platform: 'qq', type: 'group' }, + }, + { _type: 'member', platformId: 'u1', accountName: 'Alice' }, + { + _type: 'message', + sender: 'u1', + accountName: 'Alice', + timestamp: 2000, + type: 0, + content: 'new incremental message', + platformMessageId: 'm1', + }, + ] + fs.writeFileSync(filePath, rows.map((row) => JSON.stringify(row)).join('\n'), 'utf-8') +} + +test('incrementalImport raises the data directory gate after successful writes', async () => { + const root = makeTempDir() + const pathProvider = createPathProvider(root) + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.openRawSessionDatabase('existing', { create: true, initializeChatTables: true }) + db.prepare('INSERT INTO meta (name, platform, type, imported_at) VALUES (?, ?, ?, ?)').run( + 'Existing Chat', + 'qq', + 'group', + 1000 + ) + db.prepare('INSERT INTO member (platform_id, account_name) VALUES (?, ?)').run('u0', 'Existing User') + db.close() + + const filePath = path.join(root, 'incremental.jsonl') + writeIncrementalJsonl(filePath) + + const result = await incrementalImport(manager, 'existing', filePath) + + assert.equal(result.success, true) + assert.equal(result.newMessageCount, 1) + const meta = readDataDirCompatibilityMeta(pathProvider.getUserDataDir()) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) diff --git a/apps/cli/src/import/stream-import.ts b/apps/cli/src/import/stream-import.ts index 88f7d12a..3d8ca633 100644 --- a/apps/cli/src/import/stream-import.ts +++ b/apps/cli/src/import/stream-import.ts @@ -6,12 +6,10 @@ * nickname history, FTS, format fallback). */ -import * as fs from 'fs' -import * as path from 'path' import type { DatabaseManager } from '@openchatlab/node-runtime' import { + DataDirCompatibilityError, streamingImport, - openBetterSqliteDatabase, analyzeNewImport as sharedAnalyzeNewImport, analyzeIncrementalImport as sharedAnalyzeIncremental, incrementalImport as sharedIncrementalImport, @@ -27,7 +25,6 @@ import type { ImportOptions, AnalyzeNewImportResult, } from '@openchatlab/node-runtime' -import { CHAT_DB_TABLES } from '@openchatlab/core' import { detectFormat as parserDetectFormat, detectAllFormats, @@ -65,31 +62,13 @@ function generateSessionId(): string { return `chat_${ts}_${rand}` } -function resolveNativeBinding(dbManager: DatabaseManager): string | undefined { - return (dbManager as any).nativeBinding -} - function buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): StreamImportDeps { - const nativeBinding = resolveNativeBinding(dbManager) return { openDatabase(sessionId: string) { - const dbPath = dbManager.getDbPath(sessionId) - const dir = path.dirname(dbPath) - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) - const db = openBetterSqliteDatabase(dbPath, { nativeBinding }) - db.exec(CHAT_DB_TABLES) - return db + return dbManager.openRawSessionDatabase(sessionId, { create: true, initializeChatTables: true }) }, deleteDatabase(sessionId: string) { - const dbPath = dbManager.getDbPath(sessionId) - for (const suffix of ['', '-wal', '-shm']) { - try { - const p = dbPath + suffix - if (fs.existsSync(p)) fs.unlinkSync(p) - } catch { - /* ignore */ - } - } + dbManager.deleteSessionDatabaseFiles(sessionId) }, onProgress: onProgress ?? (() => {}), postImportHook(db) { @@ -103,6 +82,10 @@ function buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportPr } } +function deleteSessionDatabase(dbManager: DatabaseManager, sessionId: string): void { + dbManager.deleteSessionDatabaseFiles(sessionId) +} + /** * High-performance streaming import: parse a file and write to DB * with batched transactions, deferred indexes, and FTS. @@ -157,20 +140,38 @@ export async function streamImport( : () => {} const deps = buildStreamImportDeps(dbManager, progressAdapter) - return streamingImport(filePath, deps, formatOptions) + const result = await streamingImport(filePath, deps, formatOptions) + if (!result.success || !result.sessionId) return result + + try { + dbManager.raiseCurrentChatDbCompatibilityGate() + } catch (error) { + deleteSessionDatabase(dbManager, result.sessionId) + return { + success: false, + error: error instanceof Error ? error.message : String(error), + diagnostics: result.diagnostics, + } + } + + return result } // ==================== Incremental import ==================== -function buildIncrementalDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): IncrementalImportDeps { - const nativeBinding = resolveNativeBinding(dbManager) +function buildIncrementalDeps( + dbManager: DatabaseManager, + onProgress?: ImportProgressCallback, + onCompatibilityError?: (error: DataDirCompatibilityError) => void +): IncrementalImportDeps { return { openDatabase(sessionId: string, readonly?: boolean) { - const dbPath = dbManager.getDbPath(sessionId) - if (!fs.existsSync(dbPath)) { - throw new Error(`Session database not found: ${sessionId}`) + try { + return dbManager.openRawSessionDatabase(sessionId, { readonly: readonly ?? false }) + } catch (error) { + if (error instanceof DataDirCompatibilityError) onCompatibilityError?.(error) + throw error } - return openBetterSqliteDatabase(dbPath, { readonly: readonly ?? false, nativeBinding }) }, onProgress: onProgress ?? (() => {}), postImportHook(db) { @@ -190,7 +191,30 @@ export async function incrementalImport( options?: ImportOptions & { onProgress?: ImportProgressCallback } ): Promise { const { onProgress, ...importOpts } = options || {} - return sharedIncrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress), importOpts) + let compatibilityError: DataDirCompatibilityError | null = null + const result = await sharedIncrementalImport( + sessionId, + filePath, + buildIncrementalDeps(dbManager, onProgress, (error) => { + compatibilityError = error + }), + importOpts + ) + if (compatibilityError) throw compatibilityError + if (!result.success) return result + + try { + dbManager.raiseCurrentChatDbCompatibilityGate() + } catch (error) { + return { + ...result, + success: false, + newMessageCount: 0, + error: error instanceof Error ? error.message : String(error), + } + } + + return result } export async function analyzeIncrementalImport( diff --git a/apps/cli/src/sync/adapters.test.ts b/apps/cli/src/sync/adapters.test.ts new file mode 100644 index 00000000..9b1bb695 --- /dev/null +++ b/apps/cli/src/sync/adapters.test.ts @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, DataDirCompatibilityError, raiseDataDirMinRuntimeVersion } from '@openchatlab/node-runtime' +import { DirectImporter } from './adapters' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-sync-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('DirectImporter.sessionExists preserves session DBs when compatibility gate blocks access', () => { + const root = makeTempDir() + const pathProvider = createPathProvider(root) + const dbDir = pathProvider.getDatabaseDir() + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'blocked.db') + fs.writeFileSync(dbPath, 'sqlite content is not opened before the compatibility check', 'utf-8') + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: '0.26.0', + dataCompatibilityVersion: 2, + reason: 'future-schema', + runtime: { version: '0.26.0', kind: 'desktop' }, + module: 'future-migration', + now: () => 1780830000, + }) + + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + const importer = new DirectImporter(manager) + + assert.throws( + () => importer.sessionExists('blocked'), + (error) => error instanceof DataDirCompatibilityError && error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' + ) + assert.equal(fs.existsSync(dbPath), true) +}) diff --git a/apps/cli/src/sync/adapters.ts b/apps/cli/src/sync/adapters.ts index c17a1391..05672ca3 100644 --- a/apps/cli/src/sync/adapters.ts +++ b/apps/cli/src/sync/adapters.ts @@ -14,10 +14,9 @@ import type { HttpFetcher, DataImporter, SyncNotifier, ImportResult, FetchParams import { NOOP_LOGGER } from '@openchatlab/sync' import { buildPullUrl } from '@openchatlab/sync' import type { DatabaseManager } from '@openchatlab/node-runtime' -import { openBetterSqliteDatabase } from '@openchatlab/node-runtime' +import { DataDirCompatibilityError } from '@openchatlab/node-runtime' import { parseFile } from '../import/chatlab-reader' import { importData } from '../import/importer' -import { resolveCliPath } from '../paths' function getTempFilePath(ext: string): string { const id = crypto.randomBytes(8).toString('hex') @@ -51,21 +50,12 @@ export class NodeFetcher implements HttpFetcher { // ==================== DirectImporter ==================== -function resolveNativeBinding(): string | undefined { - if (process.versions.electron) return undefined - const nativePath = resolveCliPath('native/better_sqlite3.node') - if (fs.existsSync(nativePath)) return nativePath - return undefined -} - export class DirectImporter implements DataImporter { private dbManager: DatabaseManager - private nativeBinding: string | undefined private logger: SyncLogger constructor(dbManager: DatabaseManager, logger?: SyncLogger) { this.dbManager = dbManager - this.nativeBinding = resolveNativeBinding() this.logger = logger ?? NOOP_LOGGER } @@ -73,7 +63,7 @@ export class DirectImporter implements DataImporter { const dbPath = this.dbManager.getDbPath(sessionId) if (!fs.existsSync(dbPath)) return false try { - const db = openBetterSqliteDatabase(dbPath, { readonly: true, nativeBinding: this.nativeBinding }) + const db = this.dbManager.openRawSessionDatabase(sessionId, { readonly: true }) const row = db .prepare("SELECT COUNT(*) as cnt FROM sqlite_master WHERE type='table' AND name='message'") .get() as { cnt: number } @@ -88,7 +78,9 @@ export class DirectImporter implements DataImporter { return false } return true - } catch { + } catch (error) { + if (error instanceof DataDirCompatibilityError) throw error + this.logger.warn(`[DirectImporter] Cannot validate DB file: ${sessionId}, removing`) try { fs.unlinkSync(dbPath) @@ -119,7 +111,6 @@ export class DirectImporter implements DataImporter { this.dbManager.close(sessionId) const result = await importData(this.dbManager, data, { sessionId, - nativeBinding: this.nativeBinding, }) if (result.success) { @@ -154,7 +145,6 @@ export class DirectImporter implements DataImporter { const result = await importData(this.dbManager, data, { sessionId: externalId, - nativeBinding: this.nativeBinding, }) if (result.success) { diff --git a/apps/desktop/main/api/adapters.ts b/apps/desktop/main/api/adapters.ts index 402c0802..c9b6d4f7 100644 --- a/apps/desktop/main/api/adapters.ts +++ b/apps/desktop/main/api/adapters.ts @@ -10,9 +10,11 @@ import * as fs from 'fs' import * as path from 'path' import * as crypto from 'crypto' import Database from 'better-sqlite3' -import { net, BrowserWindow } from 'electron' +import { net, BrowserWindow, app } from 'electron' import { getTempDir } from '../paths' import * as worker from '../worker/workerManager' +import { getPathProvider } from '../path-context' +import { assertDesktopDataDirCompatible, getDesktopAppVersion } from '../runtime-compat' import type { HttpFetcher, DataImporter, SyncNotifier, ImportResult, FetchParams, SyncLogger } from '@openchatlab/sync' import { buildPullUrl, NOOP_LOGGER } from '@openchatlab/sync' @@ -84,6 +86,8 @@ export class WorkerImporter implements DataImporter { } sessionExists(sessionId: string): boolean { + assertDesktopDataDirCompatible(getPathProvider(), getDesktopAppVersion(app.getVersion())) + const dbPath = path.join(worker.getDbDirectory(), `${sessionId}.db`) if (!fs.existsSync(dbPath)) return false try { diff --git a/apps/desktop/main/database/migrations.test.ts b/apps/desktop/main/database/migrations.test.ts index 982c99c1..bd9931ff 100644 --- a/apps/desktop/main/database/migrations.test.ts +++ b/apps/desktop/main/database/migrations.test.ts @@ -8,6 +8,8 @@ import type { PathProvider } from '@openchatlab/core' import { readDataDirCompatibilityMeta } from '@openchatlab/node-runtime/src/data-dir-compat' import { migrateDatabase } from './migrations' +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + function makeTempDir(): string { const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() return fs.mkdtempSync(path.join(baseDir, 'chatlab-desktop-migration-')) @@ -32,7 +34,7 @@ test('migrateDatabase writes data directory compatibility meta after segment sch const dbPath = path.join(root, 'data', 'databases', 'desktop-legacy.db') fs.mkdirSync(path.dirname(dbPath), { recursive: true }) - const db = new Database(dbPath) + const db = new Database(dbPath, { nativeBinding }) db.exec(` CREATE TABLE meta ( name TEXT NOT NULL, diff --git a/apps/desktop/main/database/startup-migration.test.ts b/apps/desktop/main/database/startup-migration.test.ts new file mode 100644 index 00000000..cba91487 --- /dev/null +++ b/apps/desktop/main/database/startup-migration.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import Database from 'better-sqlite3' +import type { PathProvider } from '@openchatlab/core' +import { readDataDirCompatibilityMeta } from '@openchatlab/node-runtime/src/data-dir-compat' +import { assertDesktopStartupMigrationSucceeded, repairDesktopStartupCompatibilityGate } from './startup-migration' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-startup-migration-')) +} + +function makePathProvider(root: string): PathProvider { + return { + getSystemDir: () => path.join(root, 'system'), + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'system', 'ai'), + getSettingsDir: () => path.join(root, 'system', 'settings'), + getCacheDir: () => path.join(root, 'system', 'cache'), + getTempDir: () => path.join(root, 'system', 'temp'), + getLogsDir: () => path.join(root, 'system', 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('assertDesktopStartupMigrationSucceeded aborts when startup migration fails', () => { + assert.throws( + () => + assertDesktopStartupMigrationSucceeded({ + success: false, + migratedCount: 0, + failures: [{ sessionId: 'legacy', error: 'EACCES: permission denied' }], + error: '1 database migration failed', + }), + /Database schema migration failed/ + ) +}) + +test('assertDesktopStartupMigrationSucceeded accepts successful startup migrations', () => { + assert.doesNotThrow(() => assertDesktopStartupMigrationSucceeded({ success: true, migratedCount: 1, failures: [] })) +}) + +test('repairDesktopStartupCompatibilityGate raises metadata for already migrated segment schema data', () => { + const calls: Array<{ minRuntimeVersion: string; reason: string }> = [] + + repairDesktopStartupCompatibilityGate( + { version: '0.25.1', kind: 'desktop' }, + { + hasCurrentSegmentSchemaData: () => true, + raiseDataDirMinRuntimeVersion: (_pathProvider, input) => { + calls.push({ minRuntimeVersion: input.minRuntimeVersion, reason: input.reason }) + return { + formatVersion: 1, + minRuntimeVersion: input.minRuntimeVersion, + dataCompatibilityVersion: input.dataCompatibilityVersion, + reasons: [input.reason], + updatedBy: { runtime: input.runtime.kind, module: input.module, version: input.runtime.version }, + updatedAt: 1780830000, + } + }, + pathProvider: {} as never, + } + ) + + assert.deepEqual(calls, [{ minRuntimeVersion: '0.25.1', reason: 'segment-schema' }]) +}) + +test('repairDesktopStartupCompatibilityGate writes missing metadata for existing v6 databases', () => { + const root = makeTempDir() + const dbPath = path.join(root, 'data', 'databases', 'already-v6.db') + fs.mkdirSync(path.dirname(dbPath), { recursive: true }) + + const db = new Database(dbPath, { nativeBinding }) + db.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT 6 + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Already V6', 'qq', 'group', 1000, 6); + + CREATE TABLE message ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sender_id INTEGER NOT NULL, + ts INTEGER NOT NULL, + type INTEGER NOT NULL, + content TEXT + ); + + CREATE TABLE segment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + start_ts INTEGER NOT NULL, + end_ts INTEGER NOT NULL, + message_count INTEGER DEFAULT 0, + is_manual INTEGER DEFAULT 0, + summary TEXT + ); + + CREATE TABLE message_context ( + message_id INTEGER PRIMARY KEY, + segment_id INTEGER NOT NULL, + topic_id INTEGER + ); + `) + db.close() + + repairDesktopStartupCompatibilityGate( + { version: '0.25.1', kind: 'desktop' }, + { pathProvider: makePathProvider(root), nativeBinding } + ) + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) diff --git a/apps/desktop/main/database/startup-migration.ts b/apps/desktop/main/database/startup-migration.ts new file mode 100644 index 00000000..fcf5b141 --- /dev/null +++ b/apps/desktop/main/database/startup-migration.ts @@ -0,0 +1,103 @@ +import * as fs from 'node:fs' +import * as path from 'node:path' +import Database from 'better-sqlite3' +import type { PathProvider } from '@openchatlab/core' +import { CHAT_DB_COMPATIBILITY_RAISES } from '@openchatlab/node-runtime/src/migrations/chat-db-migrations' +import { + raiseDataDirMinRuntimeVersion, + type DataDirCompatibilityMeta, + type RaiseDataDirCompatibilityInput, + type RuntimeIdentity, +} from '@openchatlab/node-runtime/src/data-dir-compat' + +export interface DesktopStartupMigrationFailure { + sessionId: string + error: string +} + +export interface DesktopStartupMigrationResult { + success: boolean + migratedCount: number + failures: DesktopStartupMigrationFailure[] + error?: string +} + +export function assertDesktopStartupMigrationSucceeded(result: DesktopStartupMigrationResult): void { + if (!result.success) { + throw new Error(formatStartupMigrationError(result)) + } +} + +export interface DesktopStartupCompatibilityGateDeps { + pathProvider: PathProvider + nativeBinding?: string + hasCurrentSegmentSchemaData?: () => boolean + raiseDataDirMinRuntimeVersion?: ( + pathProvider: PathProvider, + input: RaiseDataDirCompatibilityInput + ) => DataDirCompatibilityMeta +} + +export function repairDesktopStartupCompatibilityGate( + runtime: RuntimeIdentity, + deps: DesktopStartupCompatibilityGateDeps +): void { + const hasCurrentSegmentSchemaData = + deps.hasCurrentSegmentSchemaData ?? (() => hasCurrentSegmentSchemaDatabases(deps.pathProvider, deps.nativeBinding)) + if (!hasCurrentSegmentSchemaData()) return + + const raise = deps.raiseDataDirMinRuntimeVersion ?? raiseDataDirMinRuntimeVersion + for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { + raise(deps.pathProvider, { + minRuntimeVersion: compatibilityRaise.minRuntimeVersion, + dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, + reason: compatibilityRaise.reason, + runtime, + module: compatibilityRaise.module, + }) + } +} + +function formatStartupMigrationError(result: DesktopStartupMigrationResult): string { + const details = result.failures.map((failure) => `${failure.sessionId}: ${failure.error}`).join('\n') + return ['Database schema migration failed.', result.error, details].filter(Boolean).join('\n') +} + +function hasCurrentSegmentSchemaDatabases(pathProvider: PathProvider, nativeBinding?: string): boolean { + const dbDir = pathProvider.getDatabaseDir() + if (!fs.existsSync(dbDir)) return false + + for (const file of fs.readdirSync(dbDir)) { + if (!file.endsWith('.db')) continue + + const dbPath = path.join(dbDir, file) + let db: Database.Database | null = null + try { + db = new Database(dbPath, { readonly: true, nativeBinding }) + if (isCurrentSegmentSchemaDatabase(db)) return true + } catch (error) { + console.error(`[Database] Failed to inspect compatibility gate for ${file}:`, error) + } finally { + db?.close() + } + } + + return false +} + +function isCurrentSegmentSchemaDatabase(db: Database.Database): boolean { + const tables = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'message', 'segment', 'message_context')" + ) + .all() as Array<{ name: string }> + const tableNames = new Set(tables.map((table) => table.name)) + if (!tableNames.has('meta') || !tableNames.has('message')) return false + if (!tableNames.has('segment') || !tableNames.has('message_context')) return false + + const metaColumns = db.prepare('PRAGMA table_info(meta)').all() as Array<{ name: string }> + if (!metaColumns.some((column) => column.name === 'schema_version')) return false + + const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number } | undefined + return (version?.schema_version ?? 0) >= 6 +} diff --git a/apps/desktop/main/index.ts b/apps/desktop/main/index.ts index 6786d0ca..73673245 100644 --- a/apps/desktop/main/index.ts +++ b/apps/desktop/main/index.ts @@ -20,6 +20,10 @@ import { getAiDataDir, } from './paths' import { migrateAllDatabases, checkMigrationNeeded } from './database/core' +import { + assertDesktopStartupMigrationSucceeded, + repairDesktopStartupCompatibilityGate, +} from './database/startup-migration' import { initLocale } from './i18n' import { MigrationRunner, ALL_MIGRATIONS } from '@openchatlab/config' import { assertDesktopDataDirCompatible, getDesktopAppVersion } from './runtime-compat' @@ -146,7 +150,19 @@ class MainProcess { await initLocale() // 执行数据库 schema 迁移（确保所有数据库在 Worker 查询前已是最新 schema） - this.migrateDatabasesIfNeeded(runtime) + try { + this.migrateDatabasesIfNeeded(runtime) + } catch (error) { + console.error('[Main] Database schema migration failed:', error) + dialog.showErrorBox( + 'ChatLab Database Migration Failed', + `ChatLab cannot start because database migration did not complete safely.\n\n${ + error instanceof Error ? error.message : String(error) + }` + ) + app.quit() + return + } initProxy() // 初始化代理配置 @@ -203,19 +219,14 @@ class MainProcess { } } - // 执行数据库 schema 迁移（静默迁移） + // 执行启动期数据库 schema 迁移；失败时必须中断，避免 schema 已升级但兼容门禁未落盘。 migrateDatabasesIfNeeded(runtime: RuntimeIdentity) { - try { - const { count } = checkMigrationNeeded() - if (count > 0) { - const result = migrateAllDatabases(runtime) - if (!result.success) { - console.error('[Main] Database schema migration failed:', result.error) - } - } - } catch (error) { - console.error('[Main] Error in migrateDatabasesIfNeeded:', error) + const { count } = checkMigrationNeeded() + if (count > 0) { + assertDesktopStartupMigrationSucceeded(migrateAllDatabases(runtime)) } + + repairDesktopStartupCompatibilityGate(runtime, { pathProvider: getPathProvider() }) } // 创建主窗口 diff --git a/apps/desktop/main/internal-api.ts b/apps/desktop/main/internal-api.ts index fb1b6061..6257fd25 100644 --- a/apps/desktop/main/internal-api.ts +++ b/apps/desktop/main/internal-api.ts @@ -14,7 +14,6 @@ import { randomBytes, createHmac, timingSafeEqual } from 'crypto' import { ipcMain } from 'electron' import Fastify, { type FastifyInstance, type FastifyError, type FastifyRequest, type FastifyReply } from 'fastify' import type { PathProvider } from '@openchatlab/core' -import { CHAT_DB_TABLES } from '@openchatlab/core' import { DatabaseManager, createDatabaseManagerAdapter, @@ -22,7 +21,7 @@ import { CustomProviderStore, CustomModelStore, MergeSessionCache, - openBetterSqliteDatabase, + raiseChatDbCompatibilityGate, streamingImport, } from '@openchatlab/node-runtime' import type { StreamImportDeps } from '@openchatlab/node-runtime' @@ -105,26 +104,13 @@ export async function startInternalServer(pathProvider: PathProvider): Promise { - const dbDir = pathProvider.getDatabaseDir() - if (!fs.existsSync(dbDir)) fs.mkdirSync(dbDir, { recursive: true }) - + const electronStreamImport = async (dm: DatabaseManager, filePath: string) => { const deps: StreamImportDeps = { openDatabase(sessionId: string) { - const dbPath = path.join(dbDir, `${sessionId}.db`) - const adapter = openBetterSqliteDatabase(dbPath, { readonly: false }) - adapter.exec(CHAT_DB_TABLES) - return adapter + return dm.openRawSessionDatabase(sessionId, { create: true, initializeChatTables: true }) }, deleteDatabase(sessionId: string) { - const dbPath = path.join(dbDir, `${sessionId}.db`) - for (const suffix of ['', '-wal', '-shm']) { - try { - if (fs.existsSync(dbPath + suffix)) fs.unlinkSync(dbPath + suffix) - } catch { - /* ignore */ - } - } + dm.deleteSessionDatabaseFiles(sessionId) }, onProgress() { /* no progress for merge-triggered import */ @@ -132,6 +118,12 @@ export async function startInternalServer(pathProvider: PathProvider): Promise, externalSessionId?: string ): Promise { - return sendToWorkerWithProgress('streamImport', { filePath, formatOptions, externalSessionId }, onProgress) + assertDataDirCompatibleNow() + + const result = await sendToWorkerWithProgress( + 'streamImport', + { filePath, formatOptions, externalSessionId }, + onProgress + ) + if (!result.success || !result.sessionId) return result + + try { + raiseChatDbCompatibilityGate(getPathProvider(), { + version: getDesktopAppVersion(app.getVersion()), + kind: 'desktop', + }) + } catch (error) { + deleteImportedSessionFiles(result.sessionId) + return { + success: false, + error: error instanceof Error ? error.message : String(error), + diagnostics: result.diagnostics, + } + } + + return result +} + +function deleteImportedSessionFiles(sessionId: string): void { + const dbPath = path.join(getDbDir(), `${sessionId}.db`) + for (const suffix of ['', '-wal', '-shm']) { + try { + const p = dbPath + suffix + if (fs.existsSync(p)) fs.unlinkSync(p) + } catch { + /* ignore */ + } + } } /** @@ -992,6 +1033,8 @@ export async function incrementalImport( onProgress?: (progress: ParseProgress) => void, options?: ImportOptions ): Promise { + assertDataDirCompatibleNow() + return sendToWorkerWithProgress('incrementalImport', { sessionId, filePath, options }, onProgress) } diff --git a/packages/mcp-server/src/bin.ts b/packages/mcp-server/src/bin.ts index 26f0d535..9e820f9c 100644 --- a/packages/mcp-server/src/bin.ts +++ b/packages/mcp-server/src/bin.ts @@ -10,13 +10,14 @@ import { loadConfig } from '@openchatlab/config' import { NodePathProvider, DatabaseManager } from '@openchatlab/node-runtime' import { startMcpServer } from './server' +import { getMcpPackageVersion } from './runtime-version' function main(): void { const config = loadConfig() const userDataDir = config.data.user_data_dir || undefined const pathProvider = new NodePathProvider(userDataDir) pathProvider.ensureAllDirs() - const version = process.env.npm_package_version ?? '0.25.1' + const version = getMcpPackageVersion() const dbManager = new DatabaseManager(pathProvider, { runtime: { version, kind: 'mcp' } }) startMcpServer({ version, dbManager }).catch((err) => { diff --git a/packages/mcp-server/src/env.d.ts b/packages/mcp-server/src/env.d.ts new file mode 100644 index 00000000..51ea0e97 --- /dev/null +++ b/packages/mcp-server/src/env.d.ts @@ -0,0 +1 @@ +declare const __MCP_PACKAGE_VERSION__: string diff --git a/packages/mcp-server/src/runtime-version.test.ts b/packages/mcp-server/src/runtime-version.test.ts new file mode 100644 index 00000000..f54e764b --- /dev/null +++ b/packages/mcp-server/src/runtime-version.test.ts @@ -0,0 +1,18 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { resolveMcpPackageVersion } from './runtime-version' + +test('resolveMcpPackageVersion uses bundled package version instead of npm caller env', () => { + const previous = process.env.npm_package_version + process.env.npm_package_version = '9.9.9' + + try { + assert.equal(resolveMcpPackageVersion('0.25.1'), '0.25.1') + } finally { + if (previous === undefined) { + delete process.env.npm_package_version + } else { + process.env.npm_package_version = previous + } + } +}) diff --git a/packages/mcp-server/src/runtime-version.ts b/packages/mcp-server/src/runtime-version.ts new file mode 100644 index 00000000..6c5f5310 --- /dev/null +++ b/packages/mcp-server/src/runtime-version.ts @@ -0,0 +1,8 @@ +export function resolveMcpPackageVersion(bundledVersion?: string): string { + const normalizedVersion = typeof bundledVersion === 'string' ? bundledVersion.trim() : '' + return normalizedVersion || '0.0.0-dev' +} + +export function getMcpPackageVersion(): string { + return resolveMcpPackageVersion(typeof __MCP_PACKAGE_VERSION__ !== 'undefined' ? __MCP_PACKAGE_VERSION__ : undefined) +} diff --git a/packages/mcp-server/tsup.config.ts b/packages/mcp-server/tsup.config.ts index f4de70ab..a819d486 100644 --- a/packages/mcp-server/tsup.config.ts +++ b/packages/mcp-server/tsup.config.ts @@ -1,4 +1,7 @@ import { defineConfig } from 'tsup' +import { readFileSync } from 'node:fs' + +const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf-8')) as { version: string } export default defineConfig({ entry: { @@ -14,6 +17,9 @@ export default defineConfig({ clean: true, target: 'node20', platform: 'node', + define: { + __MCP_PACKAGE_VERSION__: JSON.stringify(pkg.version), + }, noExternal: [/^@openchatlab\//], external: ['better-sqlite3', '@node-rs/jieba'], banner: { diff --git a/packages/node-runtime/src/data-dir-compat.test.ts b/packages/node-runtime/src/data-dir-compat.test.ts index 41d8e414..c0b54240 100644 --- a/packages/node-runtime/src/data-dir-compat.test.ts +++ b/packages/node-runtime/src/data-dir-compat.test.ts @@ -135,6 +135,29 @@ test('override bypasses only version insufficiency and emits a warning', () => { assert.match(warnings[0], new RegExp(userDataDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))) }) +test('override does not bypass malformed current runtime versions', () => { + const userDataDir = makeTempDir() + writeMeta(userDataDir, { + formatVersion: 1, + minRuntimeVersion: '0.25.1', + dataCompatibilityVersion: 1, + reasons: ['segment-schema'], + updatedBy: { runtime: 'desktop', module: 'chat-db-migration', version: '0.25.1' }, + updatedAt: 1780830000, + }) + + withOverride('1', () => { + assert.throws( + () => assertDataDirCompatible(makePathProvider(userDataDir), { version: '0.0.0-dev', kind: 'cli' }), + (error) => + error instanceof DataDirCompatibilityError && + error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' && + error.currentVersion === '0.0.0-dev' && + error.minRuntimeVersion === '0.25.1' + ) + }) +}) + test('broken JSON and invalid meta are blocked even with override', () => { const brokenDir = makeTempDir() fs.writeFileSync(path.join(brokenDir, '.chatlab-meta.json'), '{ broken', 'utf-8') diff --git a/packages/node-runtime/src/data-dir-compat.ts b/packages/node-runtime/src/data-dir-compat.ts index 61afc38f..06939638 100644 --- a/packages/node-runtime/src/data-dir-compat.ts +++ b/packages/node-runtime/src/data-dir-compat.ts @@ -90,7 +90,20 @@ export function assertDataDirCompatible( const meta = readDataDirCompatibilityMeta(userDataDir) if (!meta) return - if (!isStableSemver(runtime.version) || compareStableSemver(runtime.version, meta.minRuntimeVersion) < 0) { + if (!isStableSemver(runtime.version)) { + throw new DataDirCompatibilityError( + 'DATA_DIR_REQUIRES_NEWER_RUNTIME', + `ChatLab data directory compatibility check requires a stable runtime version; current version is ${runtime.version}.`, + { + userDataDir, + metaPath: getMetaPath(userDataDir), + currentVersion: runtime.version, + minRuntimeVersion: meta.minRuntimeVersion, + } + ) + } + + if (compareStableSemver(runtime.version, meta.minRuntimeVersion) < 0) { const env = options.env ?? process.env if (env.CHATLAB_ALLOW_INCOMPATIBLE_DATA_DIR === '1') { const warn = options.warn ?? console.warn diff --git a/packages/node-runtime/src/database-manager.test.ts b/packages/node-runtime/src/database-manager.test.ts index a3468654..48142d03 100644 --- a/packages/node-runtime/src/database-manager.test.ts +++ b/packages/node-runtime/src/database-manager.test.ts @@ -375,6 +375,145 @@ test('open blocks database access when data directory requires a newer runtime', ) }) +test('openRawSessionDatabase blocks raw access when data directory requires a newer runtime', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + fs.writeFileSync( + path.join(root, 'data', '.chatlab-meta.json'), + JSON.stringify( + { + formatVersion: 1, + minRuntimeVersion: '0.25.1', + dataCompatibilityVersion: 1, + reasons: ['segment-schema'], + updatedBy: { runtime: 'desktop', module: 'chat-db-migration', version: '0.25.1' }, + updatedAt: 1780830000, + }, + null, + 2 + ), + 'utf-8' + ) + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.0', kind: 'cli' }, + }) + + assert.throws( + () => manager.openRawSessionDatabase('blocked-raw', { create: true }), + (error) => + error instanceof DataDirCompatibilityError && + error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' && + error.minRuntimeVersion === '0.25.1' + ) + assert.equal(fs.existsSync(path.join(dbDir, 'blocked-raw.db')), false) +}) + +test('openRawSessionDatabase can initialize current chat tables for controlled import adapters', () => { + const root = makeTempDir() + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.openRawSessionDatabase('raw-created', { create: true, initializeChatTables: true }) + + const metaTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get() + const messageTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message'").get() + assert.ok(metaTable) + assert.ok(messageTable) + db.close() +}) + +test('raiseCurrentChatDbCompatibilityGate writes metadata for fresh current-schema databases', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'fresh-current.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT ${CURRENT_SCHEMA_VERSION} + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Fresh Current Chat', 'qq', 'group', 1000, ${CURRENT_SCHEMA_VERSION}); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + manager.raiseCurrentChatDbCompatibilityGate() + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) + assert.deepEqual(meta?.updatedBy, { + runtime: 'cli', + module: 'chat-db-migration', + version: '0.25.1', + }) +}) + +test('open repairs the data directory gate for existing current-schema databases', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'already-current.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT ${CURRENT_SCHEMA_VERSION} + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Already Current Chat', 'qq', 'group', 1000, ${CURRENT_SCHEMA_VERSION}); + + CREATE TABLE member ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + platform_id TEXT NOT NULL UNIQUE, + account_name TEXT + ); + + CREATE TABLE message ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sender_id INTEGER NOT NULL, + ts INTEGER NOT NULL, + type INTEGER NOT NULL, + content TEXT + ); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.open('already-current') + assert.ok(db) + manager.closeAll() + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) + test('open keeps a higher existing data directory runtime requirement after migration', () => { const root = makeTempDir() const dbDir = path.join(root, 'data', 'databases') diff --git a/packages/node-runtime/src/database-manager.ts b/packages/node-runtime/src/database-manager.ts index 96e33476..7c04022e 100644 --- a/packages/node-runtime/src/database-manager.ts +++ b/packages/node-runtime/src/database-manager.ts @@ -10,6 +10,7 @@ import * as fs from 'fs' import * as path from 'path' import type { DatabaseAdapter, PathProvider } from '@openchatlab/core' import { + CHAT_DB_TABLES, isChatSessionDb, runMigrations as coreRunMigrations, needsMigration as coreNeedsMigration, @@ -17,8 +18,13 @@ import { getSchemaVersion, } from '@openchatlab/core' import { openBetterSqliteDatabase } from './better-sqlite3-adapter' -import { assertDataDirCompatible, raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from './data-dir-compat' -import { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, type MigrationDeps } from './migrations' +import { assertDataDirCompatible, type RuntimeIdentity } from './data-dir-compat' +import { + CHAT_DB_COMPATIBILITY_RAISES, + getChatDbMigrations, + raiseChatDbCompatibilityGate, + type MigrationDeps, +} from './migrations' import { tokenizeForFts } from './nlp/fts-tokenizer' function createMigrationDeps(overrides?: MigrationDeps): MigrationDeps { @@ -35,6 +41,12 @@ interface DatabaseManagerOptions { allowMissingRuntimeForTests?: boolean } +interface OpenRawSessionDatabaseOptions { + readonly?: boolean + create?: boolean + initializeChatTables?: boolean +} + export class DatabaseManager { private cache = new Map() private nativeBinding?: string @@ -85,7 +97,9 @@ export class DatabaseManager { }) try { - if (!isChatSessionDb(readonlyAdapter) || !coreNeedsMigration(readonlyAdapter, CURRENT_SCHEMA_VERSION)) return + if (!isChatSessionDb(readonlyAdapter)) return + this.raiseCompatibilityGateIfNeeded(getSchemaVersion(readonlyAdapter)) + if (!coreNeedsMigration(readonlyAdapter, CURRENT_SCHEMA_VERSION)) return } finally { readonlyAdapter.close() } @@ -130,6 +144,8 @@ export class DatabaseManager { const migrations = getChatDbMigrations(this.migrationDeps) this.runMigrations(adapter, migrations) this.assertCompatible() + } else { + this.raiseCompatibilityGateIfNeeded(getSchemaVersion(adapter)) } this.cache.set(sessionId, adapter) @@ -184,6 +200,50 @@ export class DatabaseManager { return path.join(this.pathProvider.getDatabaseDir(), `${sessionId}.db`) } + openRawSessionDatabase(sessionId: string, options: OpenRawSessionDatabaseOptions = {}): DatabaseAdapter { + this.assertCompatible() + + const dbPath = this.getDbPath(sessionId) + if (!options.create && !fs.existsSync(dbPath)) { + throw new Error(`Session database not found: ${sessionId}`) + } + + if (!options.readonly) { + this.close(sessionId) + fs.mkdirSync(path.dirname(dbPath), { recursive: true }) + } + + const adapter = openBetterSqliteDatabase(dbPath, { + readonly: options.readonly ?? false, + nativeBinding: this.nativeBinding, + }) + + if (options.initializeChatTables) { + adapter.exec(CHAT_DB_TABLES) + } + + return adapter + } + + deleteSessionDatabaseFiles(sessionId: string): void { + this.close(sessionId) + + const dbPath = this.getDbPath(sessionId) + for (const suffix of ['', '-wal', '-shm']) { + try { + const filePath = dbPath + suffix + if (fs.existsSync(filePath)) fs.unlinkSync(filePath) + } catch { + /* ignore cleanup failures */ + } + } + } + + raiseCurrentChatDbCompatibilityGate(): void { + if (!this.runtime) return + raiseChatDbCompatibilityGate(this.pathProvider, this.runtime) + } + private assertCompatible(): void { if (!this.runtime) return assertDataDirCompatible(this.pathProvider, this.runtime) @@ -195,18 +255,26 @@ export class DatabaseManager { if (!migrated || !this.runtime) return const afterVersion = getSchemaVersion(adapter) - for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { - if (beforeVersion >= compatibilityRaise.migrationVersion || afterVersion < compatibilityRaise.migrationVersion) { - continue - } + if (shouldRaiseCompatibilityGate(beforeVersion, afterVersion)) { + this.raiseCurrentChatDbCompatibilityGate() + } + } - raiseDataDirMinRuntimeVersion(this.pathProvider, { - minRuntimeVersion: compatibilityRaise.minRuntimeVersion, - dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, - reason: compatibilityRaise.reason, - runtime: this.runtime, - module: compatibilityRaise.module, - }) + private raiseCompatibilityGateIfNeeded(schemaVersion: number): void { + if (!this.runtime) return + if (shouldRepairCompatibilityGate(schemaVersion)) { + this.raiseCurrentChatDbCompatibilityGate() } } } + +function shouldRaiseCompatibilityGate(beforeVersion: number, afterVersion: number): boolean { + return CHAT_DB_COMPATIBILITY_RAISES.some( + (compatibilityRaise) => + beforeVersion < compatibilityRaise.migrationVersion && afterVersion >= compatibilityRaise.migrationVersion + ) +} + +function shouldRepairCompatibilityGate(schemaVersion: number): boolean { + return CHAT_DB_COMPATIBILITY_RAISES.some((compatibilityRaise) => schemaVersion >= compatibilityRaise.migrationVersion) +} diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index 5e564fd8..0d73d1a6 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -86,6 +86,7 @@ export { raiseDataDirMinRuntimeVersion, readDataDirCompatibilityMeta, } from './data-dir-compat' +export { raiseChatDbCompatibilityGate } from './migrations/chat-db-migrations' export type { AssertDataDirCompatibilityOptions, DataDirCompatibilityMeta, diff --git a/packages/node-runtime/src/migrations/chat-db-migrations.ts b/packages/node-runtime/src/migrations/chat-db-migrations.ts index 4ed69a64..67cc55e7 100644 --- a/packages/node-runtime/src/migrations/chat-db-migrations.ts +++ b/packages/node-runtime/src/migrations/chat-db-migrations.ts @@ -7,7 +7,9 @@ */ import type { DatabaseAdapter } from '@openchatlab/core' +import type { PathProvider } from '@openchatlab/core' import type { Migration as CoreMigration } from '@openchatlab/core' +import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '../data-dir-compat' export interface MigrationDeps { /** FTS tokenizer — needed by v4 migration for backfilling the FTS index */ @@ -32,6 +34,18 @@ export const CHAT_DB_COMPATIBILITY_RAISES: ChatDbCompatibilityRaise[] = [ }, ] +export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime: RuntimeIdentity): void { + for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: compatibilityRaise.minRuntimeVersion, + dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, + reason: compatibilityRaise.reason, + runtime, + module: compatibilityRaise.module, + }) + } +} + /** * Build the chat DB migration list. * diff --git a/packages/node-runtime/src/migrations/index.ts b/packages/node-runtime/src/migrations/index.ts index 019d3672..bbf939bd 100644 --- a/packages/node-runtime/src/migrations/index.ts +++ b/packages/node-runtime/src/migrations/index.ts @@ -1,4 +1,4 @@ -export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations } from './chat-db-migrations' +export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './chat-db-migrations' export type { ChatDbCompatibilityRaise, MigrationDeps } from './chat-db-migrations' export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './electron-data-migration'