mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
feat(node-runtime): extract chat DB migrations and add auto-migration on writable open
- Extract migration definitions (v1-v4) to shared getChatDbMigrations() with DI for tokenizer - DatabaseManager.openWritable() auto-runs pending migrations - Electron migrations.ts becomes thin wrapper delegating to shared definitions
This commit is contained in:
1 parent
f78077655f
commit
064f3ce144
5 files changed
+218
-178
No files matched your search
@@ -9,22 +9,30 @@
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import type { DatabaseAdapter, PathProvider } from '@openchatlab/core'
|
||||
import { isChatSessionDb } from '@openchatlab/core'
|
||||
import {
|
||||
isChatSessionDb,
|
||||
runMigrations,
|
||||
needsMigration as coreNeedsMigration,
|
||||
CURRENT_SCHEMA_VERSION,
|
||||
} from '@openchatlab/core'
|
||||
import { openBetterSqliteDatabase } from './better-sqlite3-adapter'
|
||||
import { getChatDbMigrations, type MigrationDeps } from './migrations'
|
||||
|
||||
export class DatabaseManager {
|
||||
private cache = new Map<string, DatabaseAdapter>()
|
||||
private nativeBinding?: string
|
||||
private migrationDeps?: MigrationDeps
|
||||
|
||||
constructor(
|
||||
private pathProvider: PathProvider,
|
||||
options?: { nativeBinding?: string }
|
||||
options?: { nativeBinding?: string; migrationDeps?: MigrationDeps }
|
||||
) {
|
||||
this.nativeBinding = options?.nativeBinding
|
||||
this.migrationDeps = options?.migrationDeps
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开指定会话的数据库(带缓存)
|
||||
* Open a session DB (read-only by default, cached).
|
||||
*/
|
||||
open(sessionId: string, options?: { readonly?: boolean }): DatabaseAdapter | null {
|
||||
if (this.cache.has(sessionId)) {
|
||||
@@ -42,6 +50,36 @@ export class DatabaseManager {
|
||||
return adapter
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a session DB in read-write mode and auto-run pending migrations.
|
||||
* Falls back to `open(id, { readonly: false })` if migration is unnecessary.
|
||||
*/
|
||||
openWritable(sessionId: string): DatabaseAdapter | null {
|
||||
const existing = this.cache.get(sessionId)
|
||||
if (existing && !existing.readonly) return existing
|
||||
|
||||
if (existing) {
|
||||
existing.close()
|
||||
this.cache.delete(sessionId)
|
||||
}
|
||||
|
||||
const dbPath = this.getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return null
|
||||
|
||||
const adapter = openBetterSqliteDatabase(dbPath, {
|
||||
readonly: false,
|
||||
nativeBinding: this.nativeBinding,
|
||||
})
|
||||
|
||||
if (coreNeedsMigration(adapter, CURRENT_SCHEMA_VERSION)) {
|
||||
const migrations = getChatDbMigrations(this.migrationDeps)
|
||||
runMigrations(adapter, migrations)
|
||||
}
|
||||
|
||||
this.cache.set(sessionId, adapter)
|
||||
return adapter
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭指定会话的数据库连接
|
||||
*/
|
||||
|
||||
@@ -188,6 +188,10 @@ export type { PiModelConfig, BuildPiModelOptions } from './ai'
|
||||
export { fetchRemoteModels, validateApiKey } from './ai'
|
||||
export type { RemoteModel, FetchRemoteModelsResult, RemoteApiOptions } from './ai'
|
||||
|
||||
// Chat DB migrations
|
||||
export { getChatDbMigrations } from './migrations'
|
||||
export type { MigrationDeps } from './migrations'
|
||||
|
||||
// Merger orchestration
|
||||
export { checkConflictsFromSources, buildMergedOutput, serializeChatLabToJsonl } from './merger'
|
||||
export type {
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Chat session DB migration definitions (platform-agnostic).
|
||||
*
|
||||
* Extracted from electron/main/database/migrations.ts.
|
||||
* Migration scripts use only DatabaseAdapter — no Electron or Node-specific APIs.
|
||||
* Version 4 (FTS backfill) requires a tokenizer injected via MigrationDeps.
|
||||
*/
|
||||
|
||||
import type { DatabaseAdapter } from '@openchatlab/core'
|
||||
import type { Migration as CoreMigration } from '@openchatlab/core'
|
||||
|
||||
export interface MigrationDeps {
|
||||
/** FTS tokenizer — needed by v4 migration for backfilling the FTS index */
|
||||
tokenizeForFts?: (content: string) => string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the chat DB migration list.
|
||||
*
|
||||
* @param deps Optional dependencies (tokenizer for FTS backfill)
|
||||
* @returns Array of migrations compatible with core `runMigrations`
|
||||
*/
|
||||
export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] {
|
||||
return [
|
||||
{
|
||||
version: 1,
|
||||
description: 'Add owner_id column to meta',
|
||||
up: (db: DatabaseAdapter) => {
|
||||
const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }>
|
||||
if (!tableInfo.some((col) => col.name === 'owner_id')) {
|
||||
db.exec('ALTER TABLE meta ADD COLUMN owner_id TEXT')
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 2,
|
||||
description: 'Add roles, reply_to_message_id, platform_message_id columns',
|
||||
up: (db: DatabaseAdapter) => {
|
||||
const memberTableInfo = db.pragma('table_info(member)') as Array<{ name: string }>
|
||||
if (!memberTableInfo.some((col) => col.name === 'roles')) {
|
||||
db.exec("ALTER TABLE member ADD COLUMN roles TEXT DEFAULT '[]'")
|
||||
}
|
||||
|
||||
const messageTableInfo = db.pragma('table_info(message)') as Array<{ name: string }>
|
||||
|
||||
if (!messageTableInfo.some((col) => col.name === 'reply_to_message_id')) {
|
||||
db.exec('ALTER TABLE message ADD COLUMN reply_to_message_id TEXT DEFAULT NULL')
|
||||
}
|
||||
|
||||
if (!messageTableInfo.some((col) => col.name === 'platform_message_id')) {
|
||||
db.exec('ALTER TABLE message ADD COLUMN platform_message_id TEXT DEFAULT NULL')
|
||||
}
|
||||
|
||||
try {
|
||||
db.exec('CREATE INDEX IF NOT EXISTS idx_message_platform_id ON message(platform_message_id)')
|
||||
} catch {
|
||||
// Index may already exist
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 3,
|
||||
description: 'Add chat_session and message_context tables',
|
||||
up: (db: DatabaseAdapter) => {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_session (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
start_ts INTEGER NOT NULL,
|
||||
end_ts INTEGER NOT NULL,
|
||||
message_count INTEGER DEFAULT 0,
|
||||
is_manual INTEGER DEFAULT 0,
|
||||
summary TEXT
|
||||
)
|
||||
`)
|
||||
|
||||
try {
|
||||
db.exec('CREATE INDEX IF NOT EXISTS idx_session_time ON chat_session(start_ts, end_ts)')
|
||||
} catch {
|
||||
// Index may already exist
|
||||
}
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS message_context (
|
||||
message_id INTEGER PRIMARY KEY,
|
||||
session_id INTEGER NOT NULL,
|
||||
topic_id INTEGER
|
||||
)
|
||||
`)
|
||||
|
||||
try {
|
||||
db.exec('CREATE INDEX IF NOT EXISTS idx_context_session ON message_context(session_id)')
|
||||
} catch {
|
||||
// Index may already exist
|
||||
}
|
||||
|
||||
const tableInfo = db.pragma('table_info(meta)') as Array<{ name: string }>
|
||||
if (!tableInfo.some((col) => col.name === 'session_gap_threshold')) {
|
||||
db.exec('ALTER TABLE meta ADD COLUMN session_gap_threshold INTEGER')
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 4,
|
||||
description: 'Add FTS5 full-text search index',
|
||||
up: (db: DatabaseAdapter) => {
|
||||
const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get()
|
||||
if (hasTable) return
|
||||
|
||||
db.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5(
|
||||
content,
|
||||
content='',
|
||||
content_rowid=id
|
||||
)
|
||||
`)
|
||||
|
||||
const tokenize = deps?.tokenizeForFts
|
||||
if (!tokenize) return
|
||||
|
||||
const BATCH_SIZE = 5000
|
||||
const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)')
|
||||
|
||||
const countRow = db
|
||||
.prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''")
|
||||
.get() as { total: number } | undefined
|
||||
|
||||
const total = countRow?.total ?? 0
|
||||
let offset = 0
|
||||
while (offset < total) {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT id, content FROM message
|
||||
WHERE type = 0 AND content IS NOT NULL AND content != ''
|
||||
ORDER BY id ASC LIMIT ? OFFSET ?`
|
||||
)
|
||||
.all(BATCH_SIZE, offset) as Array<{ id: number; content: string }>
|
||||
|
||||
if (rows.length === 0) break
|
||||
|
||||
for (const row of rows) {
|
||||
const tokens = tokenize(row.content)
|
||||
if (tokens) {
|
||||
insertFts.run(row.id, tokens)
|
||||
}
|
||||
}
|
||||
|
||||
offset += BATCH_SIZE
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { getChatDbMigrations } from './chat-db-migrations'
|
||||
export type { MigrationDeps } from './chat-db-migrations'
|
||||
Reference in new issue
Block a user