diff --git a/apps/cli/src/import/importer.test.ts b/apps/cli/src/import/importer.test.ts new file mode 100644 index 00000000..d2dbcce8 --- /dev/null +++ b/apps/cli/src/import/importer.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, raiseDataDirMinRuntimeVersion, readDataDirCompatibilityMeta } from '@openchatlab/node-runtime' +import { importData } from './importer' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-import-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('importData raises the data directory gate after creating a current-schema database', async () => { + const root = makeTempDir() + fs.mkdirSync(path.join(root, 'data', 'databases'), { recursive: true }) + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const result = await importData( + manager, + { + meta: { name: 'Fresh Import', platform: 'qq', type: 'group' }, + members: [{ platformId: 'u1', accountName: 'Alice' }], + messages: [ + { + senderPlatformId: 'u1', + senderAccountName: 'Alice', + timestamp: 1000, + type: 0, + content: 'hello', + }, + ], + }, + { nativeBinding } + ) + + assert.equal(result.success, true) + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) + +test('importData re-checks data directory compatibility before raw database writes', async () => { + const root = makeTempDir() + fs.mkdirSync(path.join(root, 'data', 'databases'), { recursive: true }) + const pathProvider = createPathProvider(root) + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: '0.26.0', + dataCompatibilityVersion: 2, + reason: 'future-schema', + runtime: { version: '0.26.0', kind: 'desktop' }, + module: 'future-migration', + now: () => 1780830000, + }) + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const result = await importData( + manager, + { + meta: { name: 'Blocked Import', platform: 'qq', type: 'group' }, + members: [{ platformId: 'u1', accountName: 'Alice' }], + messages: [ + { + senderPlatformId: 'u1', + senderAccountName: 'Alice', + timestamp: 1000, + type: 0, + content: 'hello', + }, + ], + }, + { nativeBinding } + ) + + assert.equal(result.success, false) + assert.match(result.error ?? '', /requires runtime version 0\.26\.0 or newer/) + assert.equal(fs.readdirSync(path.join(root, 'data', 'databases')).filter((name) => name.endsWith('.db')).length, 0) +}) diff --git a/apps/cli/src/import/importer.ts b/apps/cli/src/import/importer.ts index ff4ceb14..5c9b1fd4 100644 --- a/apps/cli/src/import/importer.ts +++ b/apps/cli/src/import/importer.ts @@ -10,7 +10,7 @@ import * as crypto from 'crypto' import type { DatabaseAdapter } from '@openchatlab/core' import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey, buildMemberIdMap } from '@openchatlab/core' import type { DatabaseManager } from '@openchatlab/node-runtime' -import { openBetterSqliteDatabase, writeParseResultToDb } from '@openchatlab/node-runtime' +import { writeParseResultToDb } from '@openchatlab/node-runtime' import type { ParsedData, ImportMessage } from './chatlab-reader' export interface ImportResult { @@ -272,40 +272,30 @@ export async function importData( try { if (exists) { - dbManager.close(sessionId) - const db = openBetterSqliteDatabase(dbPath, { readonly: false, nativeBinding: options?.nativeBinding }) + const db = dbManager.openRawSessionDatabase(sessionId, { readonly: false }) + let result: { messageCount: number; memberCount: number; duplicateCount: number } try { - const result = incrementalImport(db, data, options?.onProgress) + result = incrementalImport(db, data, options?.onProgress) + } finally { db.close() - return { success: true, sessionId, created: false, ...result } - } catch (err) { - db.close() - throw err } + dbManager.raiseCurrentChatDbCompatibilityGate() + return { success: true, sessionId, created: false, ...result } } - const db = openBetterSqliteDatabase(dbPath, { nativeBinding: options?.nativeBinding }) - const result = fullImport(db, data, options?.onProgress) - db.close() + const db = dbManager.openRawSessionDatabase(sessionId, { create: true }) + let result: { messageCount: number; memberCount: number; duplicateCount: number } + try { + result = fullImport(db, data, options?.onProgress) + } finally { + db.close() + } + dbManager.raiseCurrentChatDbCompatibilityGate() return { success: true, sessionId, created: true, ...result } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err) if (!exists) { - try { - fs.unlinkSync(dbPath) - } catch { - /* ignore */ - } - try { - fs.unlinkSync(dbPath + '-wal') - } catch { - /* ignore */ - } - try { - fs.unlinkSync(dbPath + '-shm') - } catch { - /* ignore */ - } + dbManager.deleteSessionDatabaseFiles(sessionId) } return { success: false, diff --git a/apps/cli/src/import/stream-import.test.ts b/apps/cli/src/import/stream-import.test.ts new file mode 100644 index 00000000..e5beb8a5 --- /dev/null +++ b/apps/cli/src/import/stream-import.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, readDataDirCompatibilityMeta } from '@openchatlab/node-runtime' +import { incrementalImport } from './stream-import' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-stream-import-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +function writeIncrementalJsonl(filePath: string): void { + const rows = [ + { + _type: 'header', + chatlab: { version: '1.0.0', exportedAt: 1780830000 }, + meta: { name: 'Incremental Chat', platform: 'qq', type: 'group' }, + }, + { _type: 'member', platformId: 'u1', accountName: 'Alice' }, + { + _type: 'message', + sender: 'u1', + accountName: 'Alice', + timestamp: 2000, + type: 0, + content: 'new incremental message', + platformMessageId: 'm1', + }, + ] + fs.writeFileSync(filePath, rows.map((row) => JSON.stringify(row)).join('\n'), 'utf-8') +} + +test('incrementalImport raises the data directory gate after successful writes', async () => { + const root = makeTempDir() + const pathProvider = createPathProvider(root) + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.openRawSessionDatabase('existing', { create: true, initializeChatTables: true }) + db.prepare('INSERT INTO meta (name, platform, type, imported_at) VALUES (?, ?, ?, ?)').run( + 'Existing Chat', + 'qq', + 'group', + 1000 + ) + db.prepare('INSERT INTO member (platform_id, account_name) VALUES (?, ?)').run('u0', 'Existing User') + db.close() + + const filePath = path.join(root, 'incremental.jsonl') + writeIncrementalJsonl(filePath) + + const result = await incrementalImport(manager, 'existing', filePath) + + assert.equal(result.success, true) + assert.equal(result.newMessageCount, 1) + const meta = readDataDirCompatibilityMeta(pathProvider.getUserDataDir()) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) diff --git a/apps/cli/src/import/stream-import.ts b/apps/cli/src/import/stream-import.ts index 88f7d12a..3d8ca633 100644 --- a/apps/cli/src/import/stream-import.ts +++ b/apps/cli/src/import/stream-import.ts @@ -6,12 +6,10 @@ * nickname history, FTS, format fallback). */ -import * as fs from 'fs' -import * as path from 'path' import type { DatabaseManager } from '@openchatlab/node-runtime' import { + DataDirCompatibilityError, streamingImport, - openBetterSqliteDatabase, analyzeNewImport as sharedAnalyzeNewImport, analyzeIncrementalImport as sharedAnalyzeIncremental, incrementalImport as sharedIncrementalImport, @@ -27,7 +25,6 @@ import type { ImportOptions, AnalyzeNewImportResult, } from '@openchatlab/node-runtime' -import { CHAT_DB_TABLES } from '@openchatlab/core' import { detectFormat as parserDetectFormat, detectAllFormats, @@ -65,31 +62,13 @@ function generateSessionId(): string { return `chat_${ts}_${rand}` } -function resolveNativeBinding(dbManager: DatabaseManager): string | undefined { - return (dbManager as any).nativeBinding -} - function buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): StreamImportDeps { - const nativeBinding = resolveNativeBinding(dbManager) return { openDatabase(sessionId: string) { - const dbPath = dbManager.getDbPath(sessionId) - const dir = path.dirname(dbPath) - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) - const db = openBetterSqliteDatabase(dbPath, { nativeBinding }) - db.exec(CHAT_DB_TABLES) - return db + return dbManager.openRawSessionDatabase(sessionId, { create: true, initializeChatTables: true }) }, deleteDatabase(sessionId: string) { - const dbPath = dbManager.getDbPath(sessionId) - for (const suffix of ['', '-wal', '-shm']) { - try { - const p = dbPath + suffix - if (fs.existsSync(p)) fs.unlinkSync(p) - } catch { - /* ignore */ - } - } + dbManager.deleteSessionDatabaseFiles(sessionId) }, onProgress: onProgress ?? (() => {}), postImportHook(db) { @@ -103,6 +82,10 @@ function buildStreamImportDeps(dbManager: DatabaseManager, onProgress?: ImportPr } } +function deleteSessionDatabase(dbManager: DatabaseManager, sessionId: string): void { + dbManager.deleteSessionDatabaseFiles(sessionId) +} + /** * High-performance streaming import: parse a file and write to DB * with batched transactions, deferred indexes, and FTS. @@ -157,20 +140,38 @@ export async function streamImport( : () => {} const deps = buildStreamImportDeps(dbManager, progressAdapter) - return streamingImport(filePath, deps, formatOptions) + const result = await streamingImport(filePath, deps, formatOptions) + if (!result.success || !result.sessionId) return result + + try { + dbManager.raiseCurrentChatDbCompatibilityGate() + } catch (error) { + deleteSessionDatabase(dbManager, result.sessionId) + return { + success: false, + error: error instanceof Error ? error.message : String(error), + diagnostics: result.diagnostics, + } + } + + return result } // ==================== Incremental import ==================== -function buildIncrementalDeps(dbManager: DatabaseManager, onProgress?: ImportProgressCallback): IncrementalImportDeps { - const nativeBinding = resolveNativeBinding(dbManager) +function buildIncrementalDeps( + dbManager: DatabaseManager, + onProgress?: ImportProgressCallback, + onCompatibilityError?: (error: DataDirCompatibilityError) => void +): IncrementalImportDeps { return { openDatabase(sessionId: string, readonly?: boolean) { - const dbPath = dbManager.getDbPath(sessionId) - if (!fs.existsSync(dbPath)) { - throw new Error(`Session database not found: ${sessionId}`) + try { + return dbManager.openRawSessionDatabase(sessionId, { readonly: readonly ?? false }) + } catch (error) { + if (error instanceof DataDirCompatibilityError) onCompatibilityError?.(error) + throw error } - return openBetterSqliteDatabase(dbPath, { readonly: readonly ?? false, nativeBinding }) }, onProgress: onProgress ?? (() => {}), postImportHook(db) { @@ -190,7 +191,30 @@ export async function incrementalImport( options?: ImportOptions & { onProgress?: ImportProgressCallback } ): Promise { const { onProgress, ...importOpts } = options || {} - return sharedIncrementalImport(sessionId, filePath, buildIncrementalDeps(dbManager, onProgress), importOpts) + let compatibilityError: DataDirCompatibilityError | null = null + const result = await sharedIncrementalImport( + sessionId, + filePath, + buildIncrementalDeps(dbManager, onProgress, (error) => { + compatibilityError = error + }), + importOpts + ) + if (compatibilityError) throw compatibilityError + if (!result.success) return result + + try { + dbManager.raiseCurrentChatDbCompatibilityGate() + } catch (error) { + return { + ...result, + success: false, + newMessageCount: 0, + error: error instanceof Error ? error.message : String(error), + } + } + + return result } export async function analyzeIncrementalImport( diff --git a/apps/cli/src/sync/adapters.test.ts b/apps/cli/src/sync/adapters.test.ts new file mode 100644 index 00000000..9b1bb695 --- /dev/null +++ b/apps/cli/src/sync/adapters.test.ts @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import type { PathProvider } from '@openchatlab/core' +import { DatabaseManager, DataDirCompatibilityError, raiseDataDirMinRuntimeVersion } from '@openchatlab/node-runtime' +import { DirectImporter } from './adapters' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-cli-sync-')) +} + +function createPathProvider(root: string): PathProvider { + return { + getSystemDir: () => root, + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'ai'), + getSettingsDir: () => path.join(root, 'settings'), + getCacheDir: () => path.join(root, 'cache'), + getTempDir: () => path.join(root, 'temp'), + getLogsDir: () => path.join(root, 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('DirectImporter.sessionExists preserves session DBs when compatibility gate blocks access', () => { + const root = makeTempDir() + const pathProvider = createPathProvider(root) + const dbDir = pathProvider.getDatabaseDir() + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'blocked.db') + fs.writeFileSync(dbPath, 'sqlite content is not opened before the compatibility check', 'utf-8') + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: '0.26.0', + dataCompatibilityVersion: 2, + reason: 'future-schema', + runtime: { version: '0.26.0', kind: 'desktop' }, + module: 'future-migration', + now: () => 1780830000, + }) + + const manager = new DatabaseManager(pathProvider, { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + const importer = new DirectImporter(manager) + + assert.throws( + () => importer.sessionExists('blocked'), + (error) => error instanceof DataDirCompatibilityError && error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' + ) + assert.equal(fs.existsSync(dbPath), true) +}) diff --git a/apps/cli/src/sync/adapters.ts b/apps/cli/src/sync/adapters.ts index c17a1391..05672ca3 100644 --- a/apps/cli/src/sync/adapters.ts +++ b/apps/cli/src/sync/adapters.ts @@ -14,10 +14,9 @@ import type { HttpFetcher, DataImporter, SyncNotifier, ImportResult, FetchParams import { NOOP_LOGGER } from '@openchatlab/sync' import { buildPullUrl } from '@openchatlab/sync' import type { DatabaseManager } from '@openchatlab/node-runtime' -import { openBetterSqliteDatabase } from '@openchatlab/node-runtime' +import { DataDirCompatibilityError } from '@openchatlab/node-runtime' import { parseFile } from '../import/chatlab-reader' import { importData } from '../import/importer' -import { resolveCliPath } from '../paths' function getTempFilePath(ext: string): string { const id = crypto.randomBytes(8).toString('hex') @@ -51,21 +50,12 @@ export class NodeFetcher implements HttpFetcher { // ==================== DirectImporter ==================== -function resolveNativeBinding(): string | undefined { - if (process.versions.electron) return undefined - const nativePath = resolveCliPath('native/better_sqlite3.node') - if (fs.existsSync(nativePath)) return nativePath - return undefined -} - export class DirectImporter implements DataImporter { private dbManager: DatabaseManager - private nativeBinding: string | undefined private logger: SyncLogger constructor(dbManager: DatabaseManager, logger?: SyncLogger) { this.dbManager = dbManager - this.nativeBinding = resolveNativeBinding() this.logger = logger ?? NOOP_LOGGER } @@ -73,7 +63,7 @@ export class DirectImporter implements DataImporter { const dbPath = this.dbManager.getDbPath(sessionId) if (!fs.existsSync(dbPath)) return false try { - const db = openBetterSqliteDatabase(dbPath, { readonly: true, nativeBinding: this.nativeBinding }) + const db = this.dbManager.openRawSessionDatabase(sessionId, { readonly: true }) const row = db .prepare("SELECT COUNT(*) as cnt FROM sqlite_master WHERE type='table' AND name='message'") .get() as { cnt: number } @@ -88,7 +78,9 @@ export class DirectImporter implements DataImporter { return false } return true - } catch { + } catch (error) { + if (error instanceof DataDirCompatibilityError) throw error + this.logger.warn(`[DirectImporter] Cannot validate DB file: ${sessionId}, removing`) try { fs.unlinkSync(dbPath) @@ -119,7 +111,6 @@ export class DirectImporter implements DataImporter { this.dbManager.close(sessionId) const result = await importData(this.dbManager, data, { sessionId, - nativeBinding: this.nativeBinding, }) if (result.success) { @@ -154,7 +145,6 @@ export class DirectImporter implements DataImporter { const result = await importData(this.dbManager, data, { sessionId: externalId, - nativeBinding: this.nativeBinding, }) if (result.success) { diff --git a/apps/desktop/main/api/adapters.ts b/apps/desktop/main/api/adapters.ts index 402c0802..c9b6d4f7 100644 --- a/apps/desktop/main/api/adapters.ts +++ b/apps/desktop/main/api/adapters.ts @@ -10,9 +10,11 @@ import * as fs from 'fs' import * as path from 'path' import * as crypto from 'crypto' import Database from 'better-sqlite3' -import { net, BrowserWindow } from 'electron' +import { net, BrowserWindow, app } from 'electron' import { getTempDir } from '../paths' import * as worker from '../worker/workerManager' +import { getPathProvider } from '../path-context' +import { assertDesktopDataDirCompatible, getDesktopAppVersion } from '../runtime-compat' import type { HttpFetcher, DataImporter, SyncNotifier, ImportResult, FetchParams, SyncLogger } from '@openchatlab/sync' import { buildPullUrl, NOOP_LOGGER } from '@openchatlab/sync' @@ -84,6 +86,8 @@ export class WorkerImporter implements DataImporter { } sessionExists(sessionId: string): boolean { + assertDesktopDataDirCompatible(getPathProvider(), getDesktopAppVersion(app.getVersion())) + const dbPath = path.join(worker.getDbDirectory(), `${sessionId}.db`) if (!fs.existsSync(dbPath)) return false try { diff --git a/apps/desktop/main/database/migrations.test.ts b/apps/desktop/main/database/migrations.test.ts index 982c99c1..bd9931ff 100644 --- a/apps/desktop/main/database/migrations.test.ts +++ b/apps/desktop/main/database/migrations.test.ts @@ -8,6 +8,8 @@ import type { PathProvider } from '@openchatlab/core' import { readDataDirCompatibilityMeta } from '@openchatlab/node-runtime/src/data-dir-compat' import { migrateDatabase } from './migrations' +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + function makeTempDir(): string { const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() return fs.mkdtempSync(path.join(baseDir, 'chatlab-desktop-migration-')) @@ -32,7 +34,7 @@ test('migrateDatabase writes data directory compatibility meta after segment sch const dbPath = path.join(root, 'data', 'databases', 'desktop-legacy.db') fs.mkdirSync(path.dirname(dbPath), { recursive: true }) - const db = new Database(dbPath) + const db = new Database(dbPath, { nativeBinding }) db.exec(` CREATE TABLE meta ( name TEXT NOT NULL, diff --git a/apps/desktop/main/database/startup-migration.test.ts b/apps/desktop/main/database/startup-migration.test.ts new file mode 100644 index 00000000..cba91487 --- /dev/null +++ b/apps/desktop/main/database/startup-migration.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import Database from 'better-sqlite3' +import type { PathProvider } from '@openchatlab/core' +import { readDataDirCompatibilityMeta } from '@openchatlab/node-runtime/src/data-dir-compat' +import { assertDesktopStartupMigrationSucceeded, repairDesktopStartupCompatibilityGate } from './startup-migration' + +const nativeBinding = path.resolve('apps/cli/native/better_sqlite3.node') + +function makeTempDir(): string { + const baseDir = fs.existsSync('/private/tmp') ? '/private/tmp' : os.tmpdir() + return fs.mkdtempSync(path.join(baseDir, 'chatlab-startup-migration-')) +} + +function makePathProvider(root: string): PathProvider { + return { + getSystemDir: () => path.join(root, 'system'), + getUserDataDir: () => path.join(root, 'data'), + getDatabaseDir: () => path.join(root, 'data', 'databases'), + getAiDataDir: () => path.join(root, 'system', 'ai'), + getSettingsDir: () => path.join(root, 'system', 'settings'), + getCacheDir: () => path.join(root, 'system', 'cache'), + getTempDir: () => path.join(root, 'system', 'temp'), + getLogsDir: () => path.join(root, 'system', 'logs'), + getDownloadsDir: () => path.join(root, 'downloads'), + } +} + +test('assertDesktopStartupMigrationSucceeded aborts when startup migration fails', () => { + assert.throws( + () => + assertDesktopStartupMigrationSucceeded({ + success: false, + migratedCount: 0, + failures: [{ sessionId: 'legacy', error: 'EACCES: permission denied' }], + error: '1 database migration failed', + }), + /Database schema migration failed/ + ) +}) + +test('assertDesktopStartupMigrationSucceeded accepts successful startup migrations', () => { + assert.doesNotThrow(() => assertDesktopStartupMigrationSucceeded({ success: true, migratedCount: 1, failures: [] })) +}) + +test('repairDesktopStartupCompatibilityGate raises metadata for already migrated segment schema data', () => { + const calls: Array<{ minRuntimeVersion: string; reason: string }> = [] + + repairDesktopStartupCompatibilityGate( + { version: '0.25.1', kind: 'desktop' }, + { + hasCurrentSegmentSchemaData: () => true, + raiseDataDirMinRuntimeVersion: (_pathProvider, input) => { + calls.push({ minRuntimeVersion: input.minRuntimeVersion, reason: input.reason }) + return { + formatVersion: 1, + minRuntimeVersion: input.minRuntimeVersion, + dataCompatibilityVersion: input.dataCompatibilityVersion, + reasons: [input.reason], + updatedBy: { runtime: input.runtime.kind, module: input.module, version: input.runtime.version }, + updatedAt: 1780830000, + } + }, + pathProvider: {} as never, + } + ) + + assert.deepEqual(calls, [{ minRuntimeVersion: '0.25.1', reason: 'segment-schema' }]) +}) + +test('repairDesktopStartupCompatibilityGate writes missing metadata for existing v6 databases', () => { + const root = makeTempDir() + const dbPath = path.join(root, 'data', 'databases', 'already-v6.db') + fs.mkdirSync(path.dirname(dbPath), { recursive: true }) + + const db = new Database(dbPath, { nativeBinding }) + db.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT 6 + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Already V6', 'qq', 'group', 1000, 6); + + CREATE TABLE message ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sender_id INTEGER NOT NULL, + ts INTEGER NOT NULL, + type INTEGER NOT NULL, + content TEXT + ); + + CREATE TABLE segment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + start_ts INTEGER NOT NULL, + end_ts INTEGER NOT NULL, + message_count INTEGER DEFAULT 0, + is_manual INTEGER DEFAULT 0, + summary TEXT + ); + + CREATE TABLE message_context ( + message_id INTEGER PRIMARY KEY, + segment_id INTEGER NOT NULL, + topic_id INTEGER + ); + `) + db.close() + + repairDesktopStartupCompatibilityGate( + { version: '0.25.1', kind: 'desktop' }, + { pathProvider: makePathProvider(root), nativeBinding } + ) + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) diff --git a/apps/desktop/main/database/startup-migration.ts b/apps/desktop/main/database/startup-migration.ts new file mode 100644 index 00000000..fcf5b141 --- /dev/null +++ b/apps/desktop/main/database/startup-migration.ts @@ -0,0 +1,103 @@ +import * as fs from 'node:fs' +import * as path from 'node:path' +import Database from 'better-sqlite3' +import type { PathProvider } from '@openchatlab/core' +import { CHAT_DB_COMPATIBILITY_RAISES } from '@openchatlab/node-runtime/src/migrations/chat-db-migrations' +import { + raiseDataDirMinRuntimeVersion, + type DataDirCompatibilityMeta, + type RaiseDataDirCompatibilityInput, + type RuntimeIdentity, +} from '@openchatlab/node-runtime/src/data-dir-compat' + +export interface DesktopStartupMigrationFailure { + sessionId: string + error: string +} + +export interface DesktopStartupMigrationResult { + success: boolean + migratedCount: number + failures: DesktopStartupMigrationFailure[] + error?: string +} + +export function assertDesktopStartupMigrationSucceeded(result: DesktopStartupMigrationResult): void { + if (!result.success) { + throw new Error(formatStartupMigrationError(result)) + } +} + +export interface DesktopStartupCompatibilityGateDeps { + pathProvider: PathProvider + nativeBinding?: string + hasCurrentSegmentSchemaData?: () => boolean + raiseDataDirMinRuntimeVersion?: ( + pathProvider: PathProvider, + input: RaiseDataDirCompatibilityInput + ) => DataDirCompatibilityMeta +} + +export function repairDesktopStartupCompatibilityGate( + runtime: RuntimeIdentity, + deps: DesktopStartupCompatibilityGateDeps +): void { + const hasCurrentSegmentSchemaData = + deps.hasCurrentSegmentSchemaData ?? (() => hasCurrentSegmentSchemaDatabases(deps.pathProvider, deps.nativeBinding)) + if (!hasCurrentSegmentSchemaData()) return + + const raise = deps.raiseDataDirMinRuntimeVersion ?? raiseDataDirMinRuntimeVersion + for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { + raise(deps.pathProvider, { + minRuntimeVersion: compatibilityRaise.minRuntimeVersion, + dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, + reason: compatibilityRaise.reason, + runtime, + module: compatibilityRaise.module, + }) + } +} + +function formatStartupMigrationError(result: DesktopStartupMigrationResult): string { + const details = result.failures.map((failure) => `${failure.sessionId}: ${failure.error}`).join('\n') + return ['Database schema migration failed.', result.error, details].filter(Boolean).join('\n') +} + +function hasCurrentSegmentSchemaDatabases(pathProvider: PathProvider, nativeBinding?: string): boolean { + const dbDir = pathProvider.getDatabaseDir() + if (!fs.existsSync(dbDir)) return false + + for (const file of fs.readdirSync(dbDir)) { + if (!file.endsWith('.db')) continue + + const dbPath = path.join(dbDir, file) + let db: Database.Database | null = null + try { + db = new Database(dbPath, { readonly: true, nativeBinding }) + if (isCurrentSegmentSchemaDatabase(db)) return true + } catch (error) { + console.error(`[Database] Failed to inspect compatibility gate for ${file}:`, error) + } finally { + db?.close() + } + } + + return false +} + +function isCurrentSegmentSchemaDatabase(db: Database.Database): boolean { + const tables = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'message', 'segment', 'message_context')" + ) + .all() as Array<{ name: string }> + const tableNames = new Set(tables.map((table) => table.name)) + if (!tableNames.has('meta') || !tableNames.has('message')) return false + if (!tableNames.has('segment') || !tableNames.has('message_context')) return false + + const metaColumns = db.prepare('PRAGMA table_info(meta)').all() as Array<{ name: string }> + if (!metaColumns.some((column) => column.name === 'schema_version')) return false + + const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number } | undefined + return (version?.schema_version ?? 0) >= 6 +} diff --git a/apps/desktop/main/index.ts b/apps/desktop/main/index.ts index 6786d0ca..73673245 100644 --- a/apps/desktop/main/index.ts +++ b/apps/desktop/main/index.ts @@ -20,6 +20,10 @@ import { getAiDataDir, } from './paths' import { migrateAllDatabases, checkMigrationNeeded } from './database/core' +import { + assertDesktopStartupMigrationSucceeded, + repairDesktopStartupCompatibilityGate, +} from './database/startup-migration' import { initLocale } from './i18n' import { MigrationRunner, ALL_MIGRATIONS } from '@openchatlab/config' import { assertDesktopDataDirCompatible, getDesktopAppVersion } from './runtime-compat' @@ -146,7 +150,19 @@ class MainProcess { await initLocale() // 执行数据库 schema 迁移（确保所有数据库在 Worker 查询前已是最新 schema） - this.migrateDatabasesIfNeeded(runtime) + try { + this.migrateDatabasesIfNeeded(runtime) + } catch (error) { + console.error('[Main] Database schema migration failed:', error) + dialog.showErrorBox( + 'ChatLab Database Migration Failed', + `ChatLab cannot start because database migration did not complete safely.\n\n${ + error instanceof Error ? error.message : String(error) + }` + ) + app.quit() + return + } initProxy() // 初始化代理配置 @@ -203,19 +219,14 @@ class MainProcess { } } - // 执行数据库 schema 迁移（静默迁移） + // 执行启动期数据库 schema 迁移；失败时必须中断，避免 schema 已升级但兼容门禁未落盘。 migrateDatabasesIfNeeded(runtime: RuntimeIdentity) { - try { - const { count } = checkMigrationNeeded() - if (count > 0) { - const result = migrateAllDatabases(runtime) - if (!result.success) { - console.error('[Main] Database schema migration failed:', result.error) - } - } - } catch (error) { - console.error('[Main] Error in migrateDatabasesIfNeeded:', error) + const { count } = checkMigrationNeeded() + if (count > 0) { + assertDesktopStartupMigrationSucceeded(migrateAllDatabases(runtime)) } + + repairDesktopStartupCompatibilityGate(runtime, { pathProvider: getPathProvider() }) } // 创建主窗口 diff --git a/apps/desktop/main/internal-api.ts b/apps/desktop/main/internal-api.ts index fb1b6061..6257fd25 100644 --- a/apps/desktop/main/internal-api.ts +++ b/apps/desktop/main/internal-api.ts @@ -14,7 +14,6 @@ import { randomBytes, createHmac, timingSafeEqual } from 'crypto' import { ipcMain } from 'electron' import Fastify, { type FastifyInstance, type FastifyError, type FastifyRequest, type FastifyReply } from 'fastify' import type { PathProvider } from '@openchatlab/core' -import { CHAT_DB_TABLES } from '@openchatlab/core' import { DatabaseManager, createDatabaseManagerAdapter, @@ -22,7 +21,7 @@ import { CustomProviderStore, CustomModelStore, MergeSessionCache, - openBetterSqliteDatabase, + raiseChatDbCompatibilityGate, streamingImport, } from '@openchatlab/node-runtime' import type { StreamImportDeps } from '@openchatlab/node-runtime' @@ -105,26 +104,13 @@ export async function startInternalServer(pathProvider: PathProvider): Promise { - const dbDir = pathProvider.getDatabaseDir() - if (!fs.existsSync(dbDir)) fs.mkdirSync(dbDir, { recursive: true }) - + const electronStreamImport = async (dm: DatabaseManager, filePath: string) => { const deps: StreamImportDeps = { openDatabase(sessionId: string) { - const dbPath = path.join(dbDir, `${sessionId}.db`) - const adapter = openBetterSqliteDatabase(dbPath, { readonly: false }) - adapter.exec(CHAT_DB_TABLES) - return adapter + return dm.openRawSessionDatabase(sessionId, { create: true, initializeChatTables: true }) }, deleteDatabase(sessionId: string) { - const dbPath = path.join(dbDir, `${sessionId}.db`) - for (const suffix of ['', '-wal', '-shm']) { - try { - if (fs.existsSync(dbPath + suffix)) fs.unlinkSync(dbPath + suffix) - } catch { - /* ignore */ - } - } + dm.deleteSessionDatabaseFiles(sessionId) }, onProgress() { /* no progress for merge-triggered import */ @@ -132,6 +118,12 @@ export async function startInternalServer(pathProvider: PathProvider): Promise, externalSessionId?: string ): Promise { - return sendToWorkerWithProgress('streamImport', { filePath, formatOptions, externalSessionId }, onProgress) + assertDataDirCompatibleNow() + + const result = await sendToWorkerWithProgress( + 'streamImport', + { filePath, formatOptions, externalSessionId }, + onProgress + ) + if (!result.success || !result.sessionId) return result + + try { + raiseChatDbCompatibilityGate(getPathProvider(), { + version: getDesktopAppVersion(app.getVersion()), + kind: 'desktop', + }) + } catch (error) { + deleteImportedSessionFiles(result.sessionId) + return { + success: false, + error: error instanceof Error ? error.message : String(error), + diagnostics: result.diagnostics, + } + } + + return result +} + +function deleteImportedSessionFiles(sessionId: string): void { + const dbPath = path.join(getDbDir(), `${sessionId}.db`) + for (const suffix of ['', '-wal', '-shm']) { + try { + const p = dbPath + suffix + if (fs.existsSync(p)) fs.unlinkSync(p) + } catch { + /* ignore */ + } + } } /** @@ -992,6 +1033,8 @@ export async function incrementalImport( onProgress?: (progress: ParseProgress) => void, options?: ImportOptions ): Promise { + assertDataDirCompatibleNow() + return sendToWorkerWithProgress('incrementalImport', { sessionId, filePath, options }, onProgress) } diff --git a/packages/mcp-server/src/bin.ts b/packages/mcp-server/src/bin.ts index 26f0d535..9e820f9c 100644 --- a/packages/mcp-server/src/bin.ts +++ b/packages/mcp-server/src/bin.ts @@ -10,13 +10,14 @@ import { loadConfig } from '@openchatlab/config' import { NodePathProvider, DatabaseManager } from '@openchatlab/node-runtime' import { startMcpServer } from './server' +import { getMcpPackageVersion } from './runtime-version' function main(): void { const config = loadConfig() const userDataDir = config.data.user_data_dir || undefined const pathProvider = new NodePathProvider(userDataDir) pathProvider.ensureAllDirs() - const version = process.env.npm_package_version ?? '0.25.1' + const version = getMcpPackageVersion() const dbManager = new DatabaseManager(pathProvider, { runtime: { version, kind: 'mcp' } }) startMcpServer({ version, dbManager }).catch((err) => { diff --git a/packages/mcp-server/src/env.d.ts b/packages/mcp-server/src/env.d.ts new file mode 100644 index 00000000..51ea0e97 --- /dev/null +++ b/packages/mcp-server/src/env.d.ts @@ -0,0 +1 @@ +declare const __MCP_PACKAGE_VERSION__: string diff --git a/packages/mcp-server/src/runtime-version.test.ts b/packages/mcp-server/src/runtime-version.test.ts new file mode 100644 index 00000000..f54e764b --- /dev/null +++ b/packages/mcp-server/src/runtime-version.test.ts @@ -0,0 +1,18 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { resolveMcpPackageVersion } from './runtime-version' + +test('resolveMcpPackageVersion uses bundled package version instead of npm caller env', () => { + const previous = process.env.npm_package_version + process.env.npm_package_version = '9.9.9' + + try { + assert.equal(resolveMcpPackageVersion('0.25.1'), '0.25.1') + } finally { + if (previous === undefined) { + delete process.env.npm_package_version + } else { + process.env.npm_package_version = previous + } + } +}) diff --git a/packages/mcp-server/src/runtime-version.ts b/packages/mcp-server/src/runtime-version.ts new file mode 100644 index 00000000..6c5f5310 --- /dev/null +++ b/packages/mcp-server/src/runtime-version.ts @@ -0,0 +1,8 @@ +export function resolveMcpPackageVersion(bundledVersion?: string): string { + const normalizedVersion = typeof bundledVersion === 'string' ? bundledVersion.trim() : '' + return normalizedVersion || '0.0.0-dev' +} + +export function getMcpPackageVersion(): string { + return resolveMcpPackageVersion(typeof __MCP_PACKAGE_VERSION__ !== 'undefined' ? __MCP_PACKAGE_VERSION__ : undefined) +} diff --git a/packages/mcp-server/tsup.config.ts b/packages/mcp-server/tsup.config.ts index f4de70ab..a819d486 100644 --- a/packages/mcp-server/tsup.config.ts +++ b/packages/mcp-server/tsup.config.ts @@ -1,4 +1,7 @@ import { defineConfig } from 'tsup' +import { readFileSync } from 'node:fs' + +const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf-8')) as { version: string } export default defineConfig({ entry: { @@ -14,6 +17,9 @@ export default defineConfig({ clean: true, target: 'node20', platform: 'node', + define: { + __MCP_PACKAGE_VERSION__: JSON.stringify(pkg.version), + }, noExternal: [/^@openchatlab\//], external: ['better-sqlite3', '@node-rs/jieba'], banner: { diff --git a/packages/node-runtime/src/data-dir-compat.test.ts b/packages/node-runtime/src/data-dir-compat.test.ts index 41d8e414..c0b54240 100644 --- a/packages/node-runtime/src/data-dir-compat.test.ts +++ b/packages/node-runtime/src/data-dir-compat.test.ts @@ -135,6 +135,29 @@ test('override bypasses only version insufficiency and emits a warning', () => { assert.match(warnings[0], new RegExp(userDataDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))) }) +test('override does not bypass malformed current runtime versions', () => { + const userDataDir = makeTempDir() + writeMeta(userDataDir, { + formatVersion: 1, + minRuntimeVersion: '0.25.1', + dataCompatibilityVersion: 1, + reasons: ['segment-schema'], + updatedBy: { runtime: 'desktop', module: 'chat-db-migration', version: '0.25.1' }, + updatedAt: 1780830000, + }) + + withOverride('1', () => { + assert.throws( + () => assertDataDirCompatible(makePathProvider(userDataDir), { version: '0.0.0-dev', kind: 'cli' }), + (error) => + error instanceof DataDirCompatibilityError && + error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' && + error.currentVersion === '0.0.0-dev' && + error.minRuntimeVersion === '0.25.1' + ) + }) +}) + test('broken JSON and invalid meta are blocked even with override', () => { const brokenDir = makeTempDir() fs.writeFileSync(path.join(brokenDir, '.chatlab-meta.json'), '{ broken', 'utf-8') diff --git a/packages/node-runtime/src/data-dir-compat.ts b/packages/node-runtime/src/data-dir-compat.ts index 61afc38f..06939638 100644 --- a/packages/node-runtime/src/data-dir-compat.ts +++ b/packages/node-runtime/src/data-dir-compat.ts @@ -90,7 +90,20 @@ export function assertDataDirCompatible( const meta = readDataDirCompatibilityMeta(userDataDir) if (!meta) return - if (!isStableSemver(runtime.version) || compareStableSemver(runtime.version, meta.minRuntimeVersion) < 0) { + if (!isStableSemver(runtime.version)) { + throw new DataDirCompatibilityError( + 'DATA_DIR_REQUIRES_NEWER_RUNTIME', + `ChatLab data directory compatibility check requires a stable runtime version; current version is ${runtime.version}.`, + { + userDataDir, + metaPath: getMetaPath(userDataDir), + currentVersion: runtime.version, + minRuntimeVersion: meta.minRuntimeVersion, + } + ) + } + + if (compareStableSemver(runtime.version, meta.minRuntimeVersion) < 0) { const env = options.env ?? process.env if (env.CHATLAB_ALLOW_INCOMPATIBLE_DATA_DIR === '1') { const warn = options.warn ?? console.warn diff --git a/packages/node-runtime/src/database-manager.test.ts b/packages/node-runtime/src/database-manager.test.ts index a3468654..48142d03 100644 --- a/packages/node-runtime/src/database-manager.test.ts +++ b/packages/node-runtime/src/database-manager.test.ts @@ -375,6 +375,145 @@ test('open blocks database access when data directory requires a newer runtime', ) }) +test('openRawSessionDatabase blocks raw access when data directory requires a newer runtime', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + fs.writeFileSync( + path.join(root, 'data', '.chatlab-meta.json'), + JSON.stringify( + { + formatVersion: 1, + minRuntimeVersion: '0.25.1', + dataCompatibilityVersion: 1, + reasons: ['segment-schema'], + updatedBy: { runtime: 'desktop', module: 'chat-db-migration', version: '0.25.1' }, + updatedAt: 1780830000, + }, + null, + 2 + ), + 'utf-8' + ) + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.0', kind: 'cli' }, + }) + + assert.throws( + () => manager.openRawSessionDatabase('blocked-raw', { create: true }), + (error) => + error instanceof DataDirCompatibilityError && + error.code === 'DATA_DIR_REQUIRES_NEWER_RUNTIME' && + error.minRuntimeVersion === '0.25.1' + ) + assert.equal(fs.existsSync(path.join(dbDir, 'blocked-raw.db')), false) +}) + +test('openRawSessionDatabase can initialize current chat tables for controlled import adapters', () => { + const root = makeTempDir() + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.openRawSessionDatabase('raw-created', { create: true, initializeChatTables: true }) + + const metaTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get() + const messageTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message'").get() + assert.ok(metaTable) + assert.ok(messageTable) + db.close() +}) + +test('raiseCurrentChatDbCompatibilityGate writes metadata for fresh current-schema databases', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'fresh-current.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT ${CURRENT_SCHEMA_VERSION} + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Fresh Current Chat', 'qq', 'group', 1000, ${CURRENT_SCHEMA_VERSION}); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + manager.raiseCurrentChatDbCompatibilityGate() + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) + assert.deepEqual(meta?.updatedBy, { + runtime: 'cli', + module: 'chat-db-migration', + version: '0.25.1', + }) +}) + +test('open repairs the data directory gate for existing current-schema databases', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'already-current.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(` + CREATE TABLE meta ( + name TEXT NOT NULL, + platform TEXT NOT NULL, + type TEXT NOT NULL, + imported_at INTEGER NOT NULL, + schema_version INTEGER DEFAULT ${CURRENT_SCHEMA_VERSION} + ); + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('Already Current Chat', 'qq', 'group', 1000, ${CURRENT_SCHEMA_VERSION}); + + CREATE TABLE member ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + platform_id TEXT NOT NULL UNIQUE, + account_name TEXT + ); + + CREATE TABLE message ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sender_id INTEGER NOT NULL, + ts INTEGER NOT NULL, + type INTEGER NOT NULL, + content TEXT + ); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.25.1', kind: 'cli' }, + }) + + const db = manager.open('already-current') + assert.ok(db) + manager.closeAll() + + const meta = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(meta?.minRuntimeVersion, '0.25.1') + assert.equal(meta?.dataCompatibilityVersion, 1) + assert.deepEqual(meta?.reasons, ['segment-schema']) +}) + test('open keeps a higher existing data directory runtime requirement after migration', () => { const root = makeTempDir() const dbDir = path.join(root, 'data', 'databases') diff --git a/packages/node-runtime/src/database-manager.ts b/packages/node-runtime/src/database-manager.ts index 96e33476..7c04022e 100644 --- a/packages/node-runtime/src/database-manager.ts +++ b/packages/node-runtime/src/database-manager.ts @@ -10,6 +10,7 @@ import * as fs from 'fs' import * as path from 'path' import type { DatabaseAdapter, PathProvider } from '@openchatlab/core' import { + CHAT_DB_TABLES, isChatSessionDb, runMigrations as coreRunMigrations, needsMigration as coreNeedsMigration, @@ -17,8 +18,13 @@ import { getSchemaVersion, } from '@openchatlab/core' import { openBetterSqliteDatabase } from './better-sqlite3-adapter' -import { assertDataDirCompatible, raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from './data-dir-compat' -import { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, type MigrationDeps } from './migrations' +import { assertDataDirCompatible, type RuntimeIdentity } from './data-dir-compat' +import { + CHAT_DB_COMPATIBILITY_RAISES, + getChatDbMigrations, + raiseChatDbCompatibilityGate, + type MigrationDeps, +} from './migrations' import { tokenizeForFts } from './nlp/fts-tokenizer' function createMigrationDeps(overrides?: MigrationDeps): MigrationDeps { @@ -35,6 +41,12 @@ interface DatabaseManagerOptions { allowMissingRuntimeForTests?: boolean } +interface OpenRawSessionDatabaseOptions { + readonly?: boolean + create?: boolean + initializeChatTables?: boolean +} + export class DatabaseManager { private cache = new Map() private nativeBinding?: string @@ -85,7 +97,9 @@ export class DatabaseManager { }) try { - if (!isChatSessionDb(readonlyAdapter) || !coreNeedsMigration(readonlyAdapter, CURRENT_SCHEMA_VERSION)) return + if (!isChatSessionDb(readonlyAdapter)) return + this.raiseCompatibilityGateIfNeeded(getSchemaVersion(readonlyAdapter)) + if (!coreNeedsMigration(readonlyAdapter, CURRENT_SCHEMA_VERSION)) return } finally { readonlyAdapter.close() } @@ -130,6 +144,8 @@ export class DatabaseManager { const migrations = getChatDbMigrations(this.migrationDeps) this.runMigrations(adapter, migrations) this.assertCompatible() + } else { + this.raiseCompatibilityGateIfNeeded(getSchemaVersion(adapter)) } this.cache.set(sessionId, adapter) @@ -184,6 +200,50 @@ export class DatabaseManager { return path.join(this.pathProvider.getDatabaseDir(), `${sessionId}.db`) } + openRawSessionDatabase(sessionId: string, options: OpenRawSessionDatabaseOptions = {}): DatabaseAdapter { + this.assertCompatible() + + const dbPath = this.getDbPath(sessionId) + if (!options.create && !fs.existsSync(dbPath)) { + throw new Error(`Session database not found: ${sessionId}`) + } + + if (!options.readonly) { + this.close(sessionId) + fs.mkdirSync(path.dirname(dbPath), { recursive: true }) + } + + const adapter = openBetterSqliteDatabase(dbPath, { + readonly: options.readonly ?? false, + nativeBinding: this.nativeBinding, + }) + + if (options.initializeChatTables) { + adapter.exec(CHAT_DB_TABLES) + } + + return adapter + } + + deleteSessionDatabaseFiles(sessionId: string): void { + this.close(sessionId) + + const dbPath = this.getDbPath(sessionId) + for (const suffix of ['', '-wal', '-shm']) { + try { + const filePath = dbPath + suffix + if (fs.existsSync(filePath)) fs.unlinkSync(filePath) + } catch { + /* ignore cleanup failures */ + } + } + } + + raiseCurrentChatDbCompatibilityGate(): void { + if (!this.runtime) return + raiseChatDbCompatibilityGate(this.pathProvider, this.runtime) + } + private assertCompatible(): void { if (!this.runtime) return assertDataDirCompatible(this.pathProvider, this.runtime) @@ -195,18 +255,26 @@ export class DatabaseManager { if (!migrated || !this.runtime) return const afterVersion = getSchemaVersion(adapter) - for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { - if (beforeVersion >= compatibilityRaise.migrationVersion || afterVersion < compatibilityRaise.migrationVersion) { - continue - } + if (shouldRaiseCompatibilityGate(beforeVersion, afterVersion)) { + this.raiseCurrentChatDbCompatibilityGate() + } + } - raiseDataDirMinRuntimeVersion(this.pathProvider, { - minRuntimeVersion: compatibilityRaise.minRuntimeVersion, - dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, - reason: compatibilityRaise.reason, - runtime: this.runtime, - module: compatibilityRaise.module, - }) + private raiseCompatibilityGateIfNeeded(schemaVersion: number): void { + if (!this.runtime) return + if (shouldRepairCompatibilityGate(schemaVersion)) { + this.raiseCurrentChatDbCompatibilityGate() } } } + +function shouldRaiseCompatibilityGate(beforeVersion: number, afterVersion: number): boolean { + return CHAT_DB_COMPATIBILITY_RAISES.some( + (compatibilityRaise) => + beforeVersion < compatibilityRaise.migrationVersion && afterVersion >= compatibilityRaise.migrationVersion + ) +} + +function shouldRepairCompatibilityGate(schemaVersion: number): boolean { + return CHAT_DB_COMPATIBILITY_RAISES.some((compatibilityRaise) => schemaVersion >= compatibilityRaise.migrationVersion) +} diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index 5e564fd8..0d73d1a6 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -86,6 +86,7 @@ export { raiseDataDirMinRuntimeVersion, readDataDirCompatibilityMeta, } from './data-dir-compat' +export { raiseChatDbCompatibilityGate } from './migrations/chat-db-migrations' export type { AssertDataDirCompatibilityOptions, DataDirCompatibilityMeta, diff --git a/packages/node-runtime/src/migrations/chat-db-migrations.ts b/packages/node-runtime/src/migrations/chat-db-migrations.ts index 4ed69a64..67cc55e7 100644 --- a/packages/node-runtime/src/migrations/chat-db-migrations.ts +++ b/packages/node-runtime/src/migrations/chat-db-migrations.ts @@ -7,7 +7,9 @@ */ import type { DatabaseAdapter } from '@openchatlab/core' +import type { PathProvider } from '@openchatlab/core' import type { Migration as CoreMigration } from '@openchatlab/core' +import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '../data-dir-compat' export interface MigrationDeps { /** FTS tokenizer — needed by v4 migration for backfilling the FTS index */ @@ -32,6 +34,18 @@ export const CHAT_DB_COMPATIBILITY_RAISES: ChatDbCompatibilityRaise[] = [ }, ] +export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime: RuntimeIdentity): void { + for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { + raiseDataDirMinRuntimeVersion(pathProvider, { + minRuntimeVersion: compatibilityRaise.minRuntimeVersion, + dataCompatibilityVersion: compatibilityRaise.dataCompatibilityVersion, + reason: compatibilityRaise.reason, + runtime, + module: compatibilityRaise.module, + }) + } +} + /** * Build the chat DB migration list. * diff --git a/packages/node-runtime/src/migrations/index.ts b/packages/node-runtime/src/migrations/index.ts index 019d3672..bbf939bd 100644 --- a/packages/node-runtime/src/migrations/index.ts +++ b/packages/node-runtime/src/migrations/index.ts @@ -1,4 +1,4 @@ -export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations } from './chat-db-migrations' +export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './chat-db-migrations' export type { ChatDbCompatibilityRaise, MigrationDeps } from './chat-db-migrations' export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './electron-data-migration'