mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
fix(desktop): isolate better-sqlite3 electron binding
This commit is contained in:
1 parent
ac662d319d
commit
5f1a9bb34e
29 files changed
+472
-87
No files matched your search
@@ -86,5 +86,8 @@ mac:
|
||||
dmg:
|
||||
artifactName: ChatLab-${version}-${arch}.${ext}
|
||||
|
||||
# 是否在构建之前重新编译原生模块
|
||||
# 是否在构建之前重新编译原生模块。
|
||||
# better-sqlite3 非 NAPI,打包必须使用 Electron ABI:electron-builder 会在共享
|
||||
# node_modules 里原地重建(CI 一次性环境无影响)。本地 build:mac/build:win 通过
|
||||
# scripts/build-app.mjs 在打包结束后自动把共享绑定恢复为 Node ABI,保证 pnpm test 不被破坏。
|
||||
npmRebuild: true
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
import { AIChatManager } from '@openchatlab/node-runtime'
|
||||
import { getPathProvider } from '../path-context'
|
||||
import { resolveDesktopNativeBinding } from '../native-sqlite'
|
||||
import { aiLogger } from './logger'
|
||||
|
||||
export type { AIChat, AIMessage, AIMessageRole, ContentBlock, TokenUsageData } from '@openchatlab/node-runtime'
|
||||
@@ -21,6 +22,7 @@ export function getManager(): AIChatManager {
|
||||
aiLogger.warn(category, message, extra)
|
||||
},
|
||||
},
|
||||
nativeBinding: resolveDesktopNativeBinding(),
|
||||
})
|
||||
}
|
||||
return manager
|
||||
@@ -74,15 +76,7 @@ export function addMessage(
|
||||
contentBlocks?: import('@openchatlab/node-runtime').ContentBlock[],
|
||||
tokenUsage?: import('@openchatlab/node-runtime').TokenUsageData
|
||||
) {
|
||||
return getManager().addMessage(
|
||||
aiChatId,
|
||||
role,
|
||||
content,
|
||||
dataKeywords,
|
||||
dataMessageCount,
|
||||
contentBlocks,
|
||||
tokenUsage
|
||||
)
|
||||
return getManager().addMessage(aiChatId, role, content, dataKeywords, dataMessageCount, contentBlocks, tokenUsage)
|
||||
}
|
||||
|
||||
export function getMessages(aiChatId: string) {
|
||||
|
||||
@@ -5,11 +5,10 @@
|
||||
* then delegates all logic to the shared @openchatlab/node-runtime module.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import { completeSimple, type PiTextContent } from '@openchatlab/node-runtime'
|
||||
import { loadSegmentMessages, getSegmentSummary, saveSegmentSummary } from '@openchatlab/core'
|
||||
import { getFastModelConfig, buildPiModel } from '../llm'
|
||||
import { getDbPath, openDatabase } from '../../database/core'
|
||||
import { openDatabase } from '../../database/core'
|
||||
import { wrapAsDatabaseAdapter } from '../../worker/core'
|
||||
import { aiLogger } from '../logger'
|
||||
import { t } from '../../i18n'
|
||||
@@ -34,8 +33,9 @@ function buildDeps(dbSessionId: string): SummaryDeps {
|
||||
},
|
||||
|
||||
saveSummary(segmentId, summary) {
|
||||
const dbPath = getDbPath(dbSessionId)
|
||||
const db = new Database(dbPath)
|
||||
// 可写打开;文件不存在时返回 null(不再隐式创建空库)
|
||||
const db = openDatabase(dbSessionId, false)
|
||||
if (!db) return
|
||||
try {
|
||||
saveSegmentSummary(wrapAsDatabaseAdapter(db), segmentId, summary)
|
||||
} finally {
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as crypto from 'crypto'
|
||||
import Database from 'better-sqlite3'
|
||||
import { net, BrowserWindow, app } from 'electron'
|
||||
import { getTempDir } from '../paths'
|
||||
import { openDatabase } from '../database/core'
|
||||
import * as worker from '../worker/workerManager'
|
||||
import { getPathProvider } from '../path-context'
|
||||
import { assertDesktopDataDirCompatible, getDesktopAppVersion } from '../runtime-compat'
|
||||
@@ -91,7 +91,8 @@ export class WorkerImporter implements DataImporter {
|
||||
const dbPath = path.join(worker.getDbDirectory(), `${sessionId}.db`)
|
||||
if (!fs.existsSync(dbPath)) return false
|
||||
try {
|
||||
const db = new Database(dbPath, { readonly: true })
|
||||
const db = openDatabase(sessionId, true)
|
||||
if (!db) return false
|
||||
const row = db
|
||||
.prepare("SELECT COUNT(*) as cnt FROM sqlite_master WHERE type='table' AND name='message'")
|
||||
.get() as { cnt: number }
|
||||
|
||||
@@ -24,6 +24,7 @@ import { migrateDatabase, needsMigration, CURRENT_SCHEMA_VERSION } from './migra
|
||||
import { getPathProvider } from '../path-context'
|
||||
import { ensureDir } from '../paths'
|
||||
import { deleteSessionCache } from '@openchatlab/node-runtime'
|
||||
import { resolveDesktopNativeBinding } from '../native-sqlite'
|
||||
|
||||
/**
|
||||
* 获取数据库目录
|
||||
@@ -61,7 +62,7 @@ export function getDbPath(sessionId: string): string {
|
||||
function createDatabase(sessionId: string): Database.Database {
|
||||
ensureDbDir()
|
||||
const dbPath = getDbPath(sessionId)
|
||||
const db = new Database(dbPath)
|
||||
const db = new Database(dbPath, { nativeBinding: resolveDesktopNativeBinding() })
|
||||
|
||||
db.pragma('journal_mode = WAL')
|
||||
|
||||
@@ -80,7 +81,7 @@ export function openDatabase(sessionId: string, readonly = true): Database.Datab
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
return null
|
||||
}
|
||||
const db = new Database(dbPath, { readonly })
|
||||
const db = new Database(dbPath, { readonly, nativeBinding: resolveDesktopNativeBinding() })
|
||||
db.pragma('journal_mode = WAL')
|
||||
return db
|
||||
}
|
||||
@@ -101,7 +102,7 @@ export function openDatabaseWithMigration(
|
||||
return null
|
||||
}
|
||||
|
||||
const db = new Database(dbPath)
|
||||
const db = new Database(dbPath, { nativeBinding: resolveDesktopNativeBinding() })
|
||||
db.pragma('journal_mode = WAL')
|
||||
|
||||
// 执行迁移
|
||||
@@ -188,7 +189,7 @@ export function renameSession(sessionId: string, newName: string): boolean {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return false
|
||||
|
||||
const db = new Database(dbPath)
|
||||
const db = new Database(dbPath, { nativeBinding: resolveDesktopNativeBinding() })
|
||||
db.pragma('journal_mode = WAL')
|
||||
try {
|
||||
coreRenameSession(new BetterSqliteAdapter(db), newName)
|
||||
@@ -231,7 +232,7 @@ export function checkMigrationNeeded(): {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
|
||||
try {
|
||||
const db = new Database(dbPath, { readonly: true })
|
||||
const db = new Database(dbPath, { readonly: true, nativeBinding: resolveDesktopNativeBinding() })
|
||||
db.pragma('journal_mode = WAL')
|
||||
|
||||
// 仅迁移聊天会话数据库:这里最小依赖是 meta + message
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
repairDesktopStartupCompatibilityGate,
|
||||
} from './database/startup-migration'
|
||||
import { initLocale } from './i18n'
|
||||
import { resolveDesktopNativeBinding } from './native-sqlite'
|
||||
import { MigrationRunner, ALL_MIGRATIONS } from '@openchatlab/config'
|
||||
import { logNativeParserStatus } from '@openchatlab/node-runtime'
|
||||
import { assertDesktopDataDirCompatible, getDesktopAppVersion } from './runtime-compat'
|
||||
@@ -231,7 +232,10 @@ class MainProcess {
|
||||
assertDesktopStartupMigrationSucceeded(migrateAllDatabases(runtime))
|
||||
}
|
||||
|
||||
repairDesktopStartupCompatibilityGate(runtime, { pathProvider: getPathProvider() })
|
||||
repairDesktopStartupCompatibilityGate(runtime, {
|
||||
pathProvider: getPathProvider(),
|
||||
nativeBinding: resolveDesktopNativeBinding(),
|
||||
})
|
||||
}
|
||||
|
||||
// 创建主窗口
|
||||
|
||||
@@ -49,6 +49,7 @@ import { aiLogger } from './ai/logger'
|
||||
import { createElectronRunAgentStream } from './ai/agent-stream-runner'
|
||||
import { createExecuteElectronAiTool } from './ai/tools/debug-executor'
|
||||
import { assertDesktopDataDirCompatible, getDesktopAppVersion } from './runtime-compat'
|
||||
import { resolveDesktopNativeBinding } from './native-sqlite'
|
||||
import { resolveModelDownloadProxyUrl } from './network/proxy'
|
||||
|
||||
export interface InternalEndpoint {
|
||||
@@ -95,8 +96,9 @@ export async function startInternalServer(pathProvider: PathProvider): Promise<I
|
||||
const token = `int_${randomBytes(32).toString('hex')}`
|
||||
const { app } = await import('electron')
|
||||
const runtime = assertDesktopDataDirCompatible(pathProvider, getDesktopAppVersion(app.getVersion()))
|
||||
const nativeBinding = resolveDesktopNativeBinding()
|
||||
|
||||
newDbManager = new DatabaseManager(pathProvider, { runtime })
|
||||
newDbManager = new DatabaseManager(pathProvider, { runtime, nativeBinding })
|
||||
const sessionAdapter = createDatabaseManagerAdapter(newDbManager)
|
||||
|
||||
const aiDataDir = pathProvider.getAiDataDir()
|
||||
@@ -115,7 +117,7 @@ export async function startInternalServer(pathProvider: PathProvider): Promise<I
|
||||
|
||||
const configStorage = createFileConfigStorage(aiDataDir)
|
||||
|
||||
const newMergeCache = new MergeSessionCache(pathProvider)
|
||||
const newMergeCache = new MergeSessionCache(pathProvider, { nativeBinding })
|
||||
newMergeCache.cleanupOrphans()
|
||||
|
||||
// 语义索引 worker client:启动 internal server 时不拉起 worker;状态/构建/检索按需 lazy start。
|
||||
@@ -123,6 +125,7 @@ export async function startInternalServer(pathProvider: PathProvider): Promise<I
|
||||
newSemanticIndexService = createSemanticIndexWorkerRuntimeClient({
|
||||
pathProvider,
|
||||
runtime,
|
||||
nativeBinding,
|
||||
sqliteVecLoadablePath: getSqliteVecLoadablePath().replace('app.asar', 'app.asar.unpacked'),
|
||||
getModelDownloadProxyUrl: resolveModelDownloadProxyUrl,
|
||||
workerEntryUrl: import.meta.url.endsWith('.ts')
|
||||
@@ -167,6 +170,7 @@ export async function startInternalServer(pathProvider: PathProvider): Promise<I
|
||||
sessionAdapter,
|
||||
pathProvider,
|
||||
runtimeIdentity: runtime,
|
||||
nativeBinding,
|
||||
getVersion: () => getDesktopAppVersion(app.getVersion()),
|
||||
mergeSessionCache: newMergeCache,
|
||||
streamImport: electronStreamImport,
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { findDesktopNativeBinding, resolveDesktopNativeBinding } from './native-sqlite'
|
||||
|
||||
const BINDING = path.join('native', 'better_sqlite3.node')
|
||||
|
||||
test('findDesktopNativeBinding walks up from dev bundle locations', () => {
|
||||
const desktopRoot = path.join('/repo', 'apps', 'desktop')
|
||||
const expected = path.join(desktopRoot, BINDING)
|
||||
const exists = (candidate: string) => candidate === expected
|
||||
|
||||
// out/main(主进程 bundle)、out/main/chunks(共享 chunk)、out/main/worker(worker bundle)
|
||||
assert.equal(findDesktopNativeBinding(path.join(desktopRoot, 'out', 'main'), exists), expected)
|
||||
assert.equal(findDesktopNativeBinding(path.join(desktopRoot, 'out', 'main', 'chunks'), exists), expected)
|
||||
assert.equal(findDesktopNativeBinding(path.join(desktopRoot, 'out', 'main', 'worker'), exists), expected)
|
||||
})
|
||||
|
||||
test('findDesktopNativeBinding returns undefined when no binding exists (packaged app)', () => {
|
||||
const asarDir = path.join('/Applications', 'ChatLab.app', 'Contents', 'Resources', 'app.asar', 'out', 'main')
|
||||
assert.equal(
|
||||
findDesktopNativeBinding(asarDir, () => false),
|
||||
undefined
|
||||
)
|
||||
})
|
||||
|
||||
test('findDesktopNativeBinding stops at the filesystem root without hanging', () => {
|
||||
let calls = 0
|
||||
const exists = () => {
|
||||
calls++
|
||||
return false
|
||||
}
|
||||
assert.equal(findDesktopNativeBinding(path.parse(process.cwd()).root, exists), undefined)
|
||||
assert.ok(calls <= 5)
|
||||
})
|
||||
|
||||
test('resolveDesktopNativeBinding returns undefined under plain Node', () => {
|
||||
// 测试始终跑在系统 Node(非 Electron):node_modules 保持 Node ABI,无需注入
|
||||
assert.equal(process.versions.electron, undefined)
|
||||
assert.equal(resolveDesktopNativeBinding(), undefined)
|
||||
})
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Desktop better-sqlite3 native binding resolver.
|
||||
*
|
||||
* The shared node_modules copy of better-sqlite3 always stays on the Node ABI
|
||||
* so `pnpm test` works out of the box. When the desktop app runs under
|
||||
* Electron in dev, every database open must instead load the Electron-ABI
|
||||
* copy at apps/desktop/native/better_sqlite3.node (prepared by
|
||||
* scripts/ensure-native.mjs).
|
||||
*
|
||||
* In the packaged app this resolver returns undefined: electron-builder
|
||||
* (npmRebuild: true) rebuilds the bundled node_modules copy for Electron, so
|
||||
* the default resolution is already correct there.
|
||||
*
|
||||
* Worker threads cannot compute this reliably from their own __dirname, so the
|
||||
* main process resolves once and passes the path through workerData.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
|
||||
const NATIVE_RELATIVE_PATH = path.join('native', 'better_sqlite3.node')
|
||||
const MAX_WALK_UP_LEVELS = 5
|
||||
|
||||
/**
|
||||
* Walk up from startDir looking for native/better_sqlite3.node.
|
||||
* Dev bundles live in apps/desktop/out/main[/chunks|/worker], so the binding
|
||||
* sits at most a few levels above; the packaged app has no such directory.
|
||||
*/
|
||||
export function findDesktopNativeBinding(
|
||||
startDir: string,
|
||||
exists: (candidate: string) => boolean = fs.existsSync
|
||||
): string | undefined {
|
||||
let dir = startDir
|
||||
for (let level = 0; level < MAX_WALK_UP_LEVELS; level++) {
|
||||
const candidate = path.join(dir, NATIVE_RELATIVE_PATH)
|
||||
if (exists(candidate)) return candidate
|
||||
const parent = path.dirname(dir)
|
||||
if (parent === dir) break
|
||||
dir = parent
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
let cachedBinding: string | undefined
|
||||
let resolved = false
|
||||
|
||||
/**
|
||||
* Resolve the Electron-ABI better-sqlite3 binding for the current process.
|
||||
* Returns undefined under plain Node (tests/tools) and in the packaged app.
|
||||
*/
|
||||
export function resolveDesktopNativeBinding(): string | undefined {
|
||||
if (resolved) return cachedBinding
|
||||
resolved = true
|
||||
|
||||
if (!process.versions.electron) {
|
||||
cachedBinding = undefined
|
||||
return cachedBinding
|
||||
}
|
||||
|
||||
cachedBinding = findDesktopNativeBinding(__dirname)
|
||||
if (!cachedBinding && !__dirname.includes('app.asar')) {
|
||||
console.warn(
|
||||
'[native-sqlite] Electron-ABI better-sqlite3 binding not found; ' +
|
||||
'run `pnpm --filter @openchatlab/desktop ensure-native` or start via `pnpm dev:desktop`.'
|
||||
)
|
||||
}
|
||||
return cachedBinding
|
||||
}
|
||||
@@ -11,14 +11,30 @@ import type { DatabaseAdapter, PreparedStatement, RunResult } from '@openchatlab
|
||||
let DB_DIR: string = ''
|
||||
let CACHE_DIR: string = ''
|
||||
let TEMP_DIR: string = ''
|
||||
// Electron-ABI better-sqlite3 绑定路径(由主进程经 workerData 注入;测试/生产为 undefined)
|
||||
let NATIVE_BINDING: string | undefined
|
||||
|
||||
// 数据库连接缓存
|
||||
const dbCache = new Map<string, Database.Database>()
|
||||
|
||||
export function initDbDir(dir: string, cacheDir?: string, tempDir?: string): void {
|
||||
export function initDbDir(dir: string, cacheDir?: string, tempDir?: string, nativeBinding?: string): void {
|
||||
DB_DIR = dir
|
||||
if (cacheDir) CACHE_DIR = cacheDir
|
||||
if (tempDir) TEMP_DIR = tempDir
|
||||
NATIVE_BINDING = nativeBinding
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开一个非缓存的数据库连接 —— worker 内唯一合法的 better-sqlite3 构造入口。
|
||||
*
|
||||
* 统一携带主进程注入的 Electron-ABI nativeBinding(否则 dev 环境会加载
|
||||
* node_modules 里的 Node ABI 绑定并触发 NODE_MODULE_VERSION 报错)并启用 WAL;
|
||||
* 额外的写优化 pragma(synchronous / cache_size)由调用方按需设置。
|
||||
*/
|
||||
export function openRawDatabase(dbPath: string, options: { readonly?: boolean } = {}): Database.Database {
|
||||
const db = new Database(dbPath, { ...options, nativeBinding: NATIVE_BINDING })
|
||||
db.pragma('journal_mode = WAL')
|
||||
return db
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -42,8 +58,7 @@ export function openDatabase(sessionId: string): Database.Database | null {
|
||||
return null
|
||||
}
|
||||
|
||||
const db = new Database(dbPath, { readonly: true })
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath, { readonly: true })
|
||||
|
||||
// 缓存连接
|
||||
dbCache.set(sessionId, db)
|
||||
|
||||
@@ -7,6 +7,7 @@ export {
|
||||
initDbDir,
|
||||
getDbPath,
|
||||
openDatabase,
|
||||
openRawDatabase,
|
||||
closeDatabase,
|
||||
closeAllDatabases,
|
||||
getDbDir,
|
||||
|
||||
@@ -83,7 +83,7 @@ import {
|
||||
} from './import'
|
||||
import { initNlpDir } from '@openchatlab/node-runtime'
|
||||
|
||||
initDbDir(workerData.dbDir, workerData.cacheDir, workerData.tempDir)
|
||||
initDbDir(workerData.dbDir, workerData.cacheDir, workerData.tempDir, workerData.nativeBinding)
|
||||
|
||||
// 初始化 NLP 词库目录
|
||||
if (workerData.nlpDir) {
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
*/
|
||||
|
||||
import * as path from 'path'
|
||||
import Database from 'better-sqlite3'
|
||||
import {
|
||||
BetterSqliteAdapter,
|
||||
analyzeIncrementalImport as sharedAnalyze,
|
||||
@@ -22,7 +21,7 @@ import type {
|
||||
ImportOptions,
|
||||
} from '@openchatlab/node-runtime'
|
||||
import { sendProgress, getDbPath } from './utils'
|
||||
import { getCacheDir } from '../core'
|
||||
import { getCacheDir, openRawDatabase } from '../core'
|
||||
import * as fs from 'fs'
|
||||
|
||||
export type { ImportOptions, IncrementalAnalyzeResult, IncrementalImportResult }
|
||||
@@ -34,8 +33,7 @@ function buildDeps(requestId: string): IncrementalImportDeps {
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
throw new Error(`Session database not found: ${sessionId}`)
|
||||
}
|
||||
const db = new Database(dbPath, { readonly })
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath, { readonly })
|
||||
if (!readonly) db.pragma('synchronous = NORMAL')
|
||||
return new BetterSqliteAdapter(db)
|
||||
},
|
||||
@@ -45,8 +43,7 @@ function buildDeps(requestId: string): IncrementalImportDeps {
|
||||
postImportHook(_db, sessionId) {
|
||||
const cacheDir = getCacheDir()
|
||||
try {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
const rawDb = new Database(dbPath)
|
||||
const rawDb = openRawDatabase(getDbPath(sessionId))
|
||||
computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, cacheDir)
|
||||
rawDb.close()
|
||||
} catch (err) {
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import Database from 'better-sqlite3'
|
||||
import type Database from 'better-sqlite3'
|
||||
import {
|
||||
BetterSqliteAdapter,
|
||||
streamingImport,
|
||||
@@ -23,6 +23,7 @@ import { sendProgress, generateSessionId, getDbPath, createDatabaseWithoutIndexe
|
||||
import {
|
||||
getCacheDir,
|
||||
getTempDir,
|
||||
openRawDatabase,
|
||||
initPerfLog,
|
||||
logPerf,
|
||||
logPerfDetail,
|
||||
@@ -48,8 +49,7 @@ function generateTempDbPath(sourceFilePath: string): string {
|
||||
}
|
||||
|
||||
function createTempDatabase(dbPath: string): Database.Database {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
db.pragma('synchronous = NORMAL')
|
||||
db.exec(TEMP_DB_SCHEMA)
|
||||
return db
|
||||
@@ -92,8 +92,7 @@ function buildStreamImportDeps(requestId: string): StreamImportDeps {
|
||||
postImportHook(_db, sessionId) {
|
||||
const cacheDir = getCacheDir()
|
||||
try {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
const rawDb = new Database(dbPath)
|
||||
const rawDb = openRawDatabase(getDbPath(sessionId))
|
||||
computeAndSetOverviewCache(new BetterSqliteAdapter(rawDb), sessionId, cacheDir)
|
||||
rawDb.close()
|
||||
} catch (err) {
|
||||
|
||||
@@ -3,9 +3,12 @@
|
||||
* 用于合并导入时的临时文件处理
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import type Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
// 直接从 dbCore 导入:core/index 会连带 perfLogger → @openchatlab/node-runtime,
|
||||
// 而本模块被纯 Node 单测导入,需保持依赖面最小。
|
||||
import { openRawDatabase } from '../core/dbCore'
|
||||
|
||||
// 在 Worker 线程中,无法直接使用 electron 的 app 模块
|
||||
// 需要通过其他方式获取临时目录
|
||||
@@ -25,9 +28,8 @@ function getTempDir(): string {
|
||||
export function createTempDatabase(): { db: Database.Database; path: string } {
|
||||
const tempDir = getTempDir()
|
||||
const tempPath = path.join(tempDir, `merge_${Date.now()}_${Math.random().toString(36).slice(2)}.db`)
|
||||
const db = new Database(tempPath)
|
||||
const db = openRawDatabase(tempPath)
|
||||
|
||||
db.pragma('journal_mode = WAL')
|
||||
db.pragma('synchronous = NORMAL')
|
||||
db.pragma('cache_size = -64000')
|
||||
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
* Import module utilities shared across stream/incremental import.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import type Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { parentPort } from 'worker_threads'
|
||||
import { getDbDir } from '../core'
|
||||
import { getDbDir, openRawDatabase } from '../core'
|
||||
import { CHAT_DB_TABLES, CHAT_DB_INDEXES } from '@openchatlab/core'
|
||||
import type { ParseProgress } from '../../parser'
|
||||
|
||||
@@ -39,9 +39,8 @@ export function createDatabaseWithoutIndexes(sessionId: string): Database.Databa
|
||||
}
|
||||
|
||||
const dbPath = getDbPath(sessionId)
|
||||
const db = new Database(dbPath)
|
||||
const db = openRawDatabase(dbPath)
|
||||
|
||||
db.pragma('journal_mode = WAL')
|
||||
db.pragma('synchronous = NORMAL')
|
||||
db.pragma('cache_size = -64000') // 64MB cache for write performance
|
||||
|
||||
|
||||
@@ -3,9 +3,8 @@
|
||||
* 查询逻辑委托给 @openchatlab/core,本模块负责数据库连接管理和成员 DDL 迁移
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import * as fs from 'fs'
|
||||
import { closeDatabase, getDbPath, getCacheDir, openDatabaseAdapter, type TimeFilter } from '../core'
|
||||
import { closeDatabase, getDbPath, getCacheDir, openRawDatabase, openDatabaseAdapter, type TimeFilter } from '../core'
|
||||
import { getCache, CACHE_KEY_OVERVIEW, type OverviewCache } from '@openchatlab/node-runtime'
|
||||
import {
|
||||
getAvailableYears as coreGetAvailableYears,
|
||||
@@ -122,8 +121,7 @@ function ensureAliasesColumn(sessionId: string): void {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return
|
||||
closeDatabase(sessionId)
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
try {
|
||||
const adapter = new BetterSqliteAdapter(db)
|
||||
if (coreEnsureAliasesColumn(adapter)) {
|
||||
@@ -140,8 +138,7 @@ export function ensureAvatarColumn(sessionId: string): void {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return
|
||||
closeDatabase(sessionId)
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
try {
|
||||
const adapter = new BetterSqliteAdapter(db)
|
||||
if (coreEnsureAvatarColumn(adapter)) {
|
||||
@@ -189,8 +186,7 @@ export function updateMemberAliases(sessionId: string, memberId: number, aliases
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return false
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
const adapter = new BetterSqliteAdapter(db)
|
||||
const result = coreUpdateMemberAliases(adapter, memberId, aliases)
|
||||
db.close()
|
||||
@@ -205,8 +201,7 @@ export function mergeMembers(sessionId: string, memberId1: number, memberId2: nu
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return false
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
const adapter = new BetterSqliteAdapter(db)
|
||||
const result = coreMergeMembers(adapter, memberId1, memberId2)
|
||||
db.close()
|
||||
@@ -221,8 +216,7 @@ export function deleteMember(sessionId: string, memberId: number): boolean {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
if (!fs.existsSync(dbPath)) return false
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
const adapter = new BetterSqliteAdapter(db)
|
||||
const result = coreDeleteMember(adapter, memberId)
|
||||
db.close()
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* This module adds session-level DB connection management.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import type Database from 'better-sqlite3'
|
||||
import { BetterSqliteAdapter } from '@openchatlab/node-runtime'
|
||||
import {
|
||||
hasFtsTable,
|
||||
@@ -15,14 +15,11 @@ import {
|
||||
insertFtsEntries as coreInsertFtsEntries,
|
||||
searchByFts as coreSearchByFts,
|
||||
} from '@openchatlab/node-runtime'
|
||||
import { getDbPath, openDatabase } from '../core'
|
||||
import { getDbPath, openRawDatabase, openDatabase } from '../core'
|
||||
|
||||
function openWritableDb(sessionId: string): Database.Database | null {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
return db
|
||||
return openRawDatabase(getDbPath(sessionId))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
* 提供数据库连接等共享功能
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import { getDbPath, closeDatabase } from '../../core'
|
||||
import type Database from 'better-sqlite3'
|
||||
import { getDbPath, openRawDatabase, closeDatabase } from '../../core'
|
||||
|
||||
// 重新导出 closeDatabase 供其他模块使用
|
||||
export { closeDatabase }
|
||||
@@ -14,11 +14,8 @@ export { closeDatabase }
|
||||
* 会话索引需要写入数据
|
||||
*/
|
||||
export function openWritableDatabase(sessionId: string): Database.Database | null {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
return db
|
||||
return openRawDatabase(getDbPath(sessionId))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
@@ -28,11 +25,8 @@ export function openWritableDatabase(sessionId: string): Database.Database | nul
|
||||
* 打开数据库(只读模式,不使用缓存)
|
||||
*/
|
||||
export function openReadonlyDatabase(sessionId: string): Database.Database | null {
|
||||
const dbPath = getDbPath(sessionId)
|
||||
try {
|
||||
const db = new Database(dbPath, { readonly: true })
|
||||
db.pragma('journal_mode = WAL')
|
||||
return db
|
||||
return openRawDatabase(getDbPath(sessionId), { readonly: true })
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -7,11 +7,11 @@
|
||||
* dbPath, aiConversationCount, and the getChatOverview cache-first pattern.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3'
|
||||
import type Database from 'better-sqlite3'
|
||||
import { BetterSqliteAdapter } from '@openchatlab/node-runtime'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { openDatabase, getDbDir, getDbPath, getCacheDir } from '../core'
|
||||
import { openDatabase, openRawDatabase, getDbDir, getDbPath, getCacheDir } from '../core'
|
||||
import {
|
||||
getCache,
|
||||
getValidatedOverviewCache,
|
||||
@@ -82,8 +82,7 @@ export function getAllSessions(): any[] {
|
||||
const dbPath = path.join(dbDir, file)
|
||||
|
||||
try {
|
||||
const db = new Database(dbPath)
|
||||
db.pragma('journal_mode = WAL')
|
||||
const db = openRawDatabase(dbPath)
|
||||
|
||||
if (!isChatSessionDb(db)) {
|
||||
db.close()
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { StreamImportResult } from './import'
|
||||
|
||||
import { getDatabaseDir, getCacheDir, getTempDir, ensureDir } from '../paths'
|
||||
import { getNlpDir } from '../nlp/dictManager'
|
||||
import { resolveDesktopNativeBinding } from '../native-sqlite'
|
||||
import { assertDesktopDataDirCompatible, getDesktopAppVersion } from '../runtime-compat'
|
||||
import { getPathProvider } from '../path-context'
|
||||
import { raiseChatDbCompatibilityGate } from '@openchatlab/node-runtime'
|
||||
@@ -133,6 +134,8 @@ export function initWorker(): void {
|
||||
tempDir: getTempDir(),
|
||||
nlpDir: getNlpDir(),
|
||||
appVersion: getDesktopAppVersion(app.getVersion()),
|
||||
// Worker threads cannot resolve the Electron-ABI binding themselves; hand it over.
|
||||
nativeBinding: resolveDesktopNativeBinding(),
|
||||
},
|
||||
})
|
||||
worker = initializedWorker
|
||||
|
||||
@@ -6,12 +6,13 @@
|
||||
"description": "ChatLab Desktop — Electron 桌面客户端",
|
||||
"main": "./out/main/index.js",
|
||||
"scripts": {
|
||||
"postinstall": "electron-rebuild -f -w better-sqlite3",
|
||||
"dev": "electron-vite dev",
|
||||
"preview": "electron-vite preview",
|
||||
"ensure-native": "node scripts/ensure-native.mjs",
|
||||
"restore-node-abi": "node scripts/restore-node-abi.mjs",
|
||||
"dev": "node scripts/ensure-native.mjs && electron-vite dev",
|
||||
"preview": "node scripts/ensure-native.mjs && electron-vite preview",
|
||||
"build": "electron-vite build",
|
||||
"build:mac": "pnpm run build && electron-builder --mac --config electron-builder.yml -p never",
|
||||
"build:win": "pnpm run build && electron-builder --win --config electron-builder.yml -p never"
|
||||
"build:mac": "node scripts/ensure-native.mjs && pnpm run build && node scripts/build-app.mjs --mac",
|
||||
"build:win": "node scripts/ensure-native.mjs && pnpm run build && node scripts/build-app.mjs --win"
|
||||
},
|
||||
"dependencies": {
|
||||
"@electron-toolkit/preload": "^3.0.1",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Run electron-builder, then always restore the shared better-sqlite3 binding
|
||||
* to the Node ABI — even when packaging fails — so a local `pnpm build:mac` /
|
||||
* `pnpm build:win` never leaves `pnpm test` broken.
|
||||
*
|
||||
* Usage: node scripts/build-app.mjs --mac [extra electron-builder args]
|
||||
*/
|
||||
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { restoreNodeAbi } from './restore-node-abi.mjs'
|
||||
|
||||
const desktopDir = dirname(dirname(fileURLToPath(import.meta.url)))
|
||||
const builderArgs = [...process.argv.slice(2), '--config', 'electron-builder.yml', '-p', 'never']
|
||||
|
||||
const result = spawnSync('pnpm', ['exec', 'electron-builder', ...builderArgs], {
|
||||
cwd: desktopDir,
|
||||
stdio: 'inherit',
|
||||
shell: process.platform === 'win32',
|
||||
})
|
||||
|
||||
const restored = restoreNodeAbi()
|
||||
const builderExitCode = result.status ?? 1
|
||||
|
||||
if (builderExitCode !== 0) process.exit(builderExitCode)
|
||||
if (!restored) process.exit(1)
|
||||
|
||||
console.error(`[desktop build] packaged app written to ${resolve(desktopDir, 'dist')}`)
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Ensure apps/desktop/native/better_sqlite3.node is a valid Electron-ABI binding.
|
||||
*
|
||||
* The shared node_modules copy of better-sqlite3 stays on the Node ABI
|
||||
* (so `pnpm test` always works); Electron dev loads this desktop-owned copy
|
||||
* instead via the nativeBinding option (see main/native-sqlite.ts).
|
||||
*
|
||||
* Verification runs the Electron binary with ELECTRON_RUN_AS_NODE so the
|
||||
* check uses the real Electron ABI. Rebuild downloads the matching Electron
|
||||
* prebuilt into an isolated temp dir (never touching shared node_modules),
|
||||
* falling back to a node-gyp source build against Electron headers.
|
||||
*/
|
||||
|
||||
import { createRequire } from 'node:module'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, copyFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const currentFile = fileURLToPath(import.meta.url)
|
||||
const desktopDir = dirname(dirname(currentFile))
|
||||
const nativePath = resolve(desktopDir, 'native/better_sqlite3.node')
|
||||
const desktopRequire = createRequire(join(desktopDir, 'package.json'))
|
||||
const useShell = process.platform === 'win32'
|
||||
|
||||
function log(message) {
|
||||
console.error(`[desktop native] ${message}`)
|
||||
}
|
||||
|
||||
function resolveElectron() {
|
||||
// Under plain Node, require('electron') returns the path to the Electron binary.
|
||||
const binaryPath = desktopRequire('electron')
|
||||
const version = desktopRequire('electron/package.json').version
|
||||
return { binaryPath, version }
|
||||
}
|
||||
|
||||
export function getElectronNativeStatus(bindingPath, electronBinaryPath) {
|
||||
if (!existsSync(bindingPath)) {
|
||||
return { ok: false, reason: 'missing', message: `Native binding not found: ${bindingPath}` }
|
||||
}
|
||||
|
||||
const result = spawnSync(
|
||||
electronBinaryPath,
|
||||
['-e', 'require(process.argv[1]); process.stdout.write(process.versions.modules)', bindingPath],
|
||||
{ encoding: 'utf8', env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' } }
|
||||
)
|
||||
|
||||
if (result.status === 0) {
|
||||
return { ok: true, reason: 'valid', abi: result.stdout.trim() }
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'invalid',
|
||||
message: (result.stderr || result.stdout || result.error?.message || 'Native binding failed to load').trim(),
|
||||
}
|
||||
}
|
||||
|
||||
function run(command, args, options) {
|
||||
const result = spawnSync(command, args, { stdio: 'inherit', shell: useShell, ...options })
|
||||
return result.status === 0
|
||||
}
|
||||
|
||||
function rebuild(electronVersion) {
|
||||
const sqliteVersion = desktopRequire('better-sqlite3/package.json').version
|
||||
log(`Fetching better-sqlite3@${sqliteVersion} for Electron ${electronVersion}...`)
|
||||
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'chatlab-desktop-native-'))
|
||||
try {
|
||||
const installOk = run(
|
||||
'npm',
|
||||
['install', `better-sqlite3@${sqliteVersion}`, '--ignore-scripts', '--no-audit', '--no-fund'],
|
||||
{ cwd: tempDir }
|
||||
)
|
||||
if (!installOk) {
|
||||
log('Failed to stage better-sqlite3 sources in temp dir')
|
||||
return false
|
||||
}
|
||||
|
||||
const pkgDir = join(tempDir, 'node_modules', 'better-sqlite3')
|
||||
const prebuiltOk = run('npx', ['--yes', 'prebuild-install', '-r', 'electron', '-t', electronVersion], {
|
||||
cwd: pkgDir,
|
||||
})
|
||||
if (!prebuiltOk) {
|
||||
log('Electron prebuilt unavailable; compiling from source (requires build toolchain)...')
|
||||
const gypOk = run(
|
||||
'npx',
|
||||
[
|
||||
'--yes',
|
||||
'node-gyp',
|
||||
'rebuild',
|
||||
'--release',
|
||||
`--runtime=electron`,
|
||||
`--target=${electronVersion}`,
|
||||
'--dist-url=https://electronjs.org/headers',
|
||||
],
|
||||
{ cwd: pkgDir }
|
||||
)
|
||||
if (!gypOk) return false
|
||||
}
|
||||
|
||||
const built = join(pkgDir, 'build', 'Release', 'better_sqlite3.node')
|
||||
if (!existsSync(built)) {
|
||||
log(`Build finished but binary not found at ${built}`)
|
||||
return false
|
||||
}
|
||||
|
||||
mkdirSync(dirname(nativePath), { recursive: true })
|
||||
copyFileSync(built, nativePath)
|
||||
return true
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const checkOnly = process.argv.includes('--check')
|
||||
const electron = resolveElectron()
|
||||
const status = getElectronNativeStatus(nativePath, electron.binaryPath)
|
||||
|
||||
if (status.ok) {
|
||||
log(`better-sqlite3 ready (Electron ABI ${status.abi})`)
|
||||
return
|
||||
}
|
||||
|
||||
if (checkOnly) {
|
||||
log(status.message)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
log(status.message)
|
||||
if (!rebuild(electron.version)) {
|
||||
log('Failed to prepare the Electron native binding')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const rebuilt = getElectronNativeStatus(nativePath, electron.binaryPath)
|
||||
if (!rebuilt.ok) {
|
||||
log(`Rebuild completed, but native binding is still unusable: ${rebuilt.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
log(`better-sqlite3 ready (Electron ABI ${rebuilt.abi})`)
|
||||
}
|
||||
|
||||
if (process.argv[1] && currentFile === resolve(process.argv[1])) {
|
||||
main()
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Restore the shared node_modules better-sqlite3 binding to the Node ABI.
|
||||
*
|
||||
* electron-builder (npmRebuild: true) rebuilds better-sqlite3 in place for
|
||||
* Electron while packaging, which would break `pnpm test` afterwards. This
|
||||
* script verifies the shared binding still loads under the current Node and,
|
||||
* if not, re-runs the package install script (prebuild-install) to bring the
|
||||
* Node prebuilt back.
|
||||
*/
|
||||
|
||||
import { createRequire } from 'node:module'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const currentFile = fileURLToPath(import.meta.url)
|
||||
const desktopDir = dirname(dirname(currentFile))
|
||||
const desktopRequire = createRequire(join(desktopDir, 'package.json'))
|
||||
|
||||
function log(message) {
|
||||
console.error(`[desktop native] ${message}`)
|
||||
}
|
||||
|
||||
function getNodeAbiStatus(bindingPath) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
['-e', 'require(process.argv[1]); process.stdout.write(process.versions.modules)', bindingPath],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
if (result.status === 0) return { ok: true, abi: result.stdout.trim() }
|
||||
return { ok: false, message: (result.stderr || result.stdout || 'binding failed to load').trim() }
|
||||
}
|
||||
|
||||
export function restoreNodeAbi() {
|
||||
const pkgDir = dirname(desktopRequire.resolve('better-sqlite3/package.json'))
|
||||
const bindingPath = join(pkgDir, 'build', 'Release', 'better_sqlite3.node')
|
||||
|
||||
const before = getNodeAbiStatus(bindingPath)
|
||||
if (before.ok) {
|
||||
log(`shared better-sqlite3 already on Node ABI ${before.abi}; nothing to restore`)
|
||||
return true
|
||||
}
|
||||
|
||||
log('shared better-sqlite3 was rebuilt for Electron by electron-builder; restoring Node ABI...')
|
||||
const rebuild = spawnSync('npm', ['rebuild', 'better-sqlite3'], {
|
||||
cwd: desktopDir,
|
||||
stdio: 'inherit',
|
||||
shell: process.platform === 'win32',
|
||||
})
|
||||
if (rebuild.status !== 0) {
|
||||
log('npm rebuild better-sqlite3 failed; run it manually before `pnpm test`')
|
||||
return false
|
||||
}
|
||||
|
||||
const after = getNodeAbiStatus(bindingPath)
|
||||
if (!after.ok) {
|
||||
log(`restore finished but binding still unusable under Node: ${after.message}`)
|
||||
return false
|
||||
}
|
||||
|
||||
log(`shared better-sqlite3 restored to Node ABI ${after.abi}`)
|
||||
return true
|
||||
}
|
||||
|
||||
if (process.argv[1] && currentFile === resolve(process.argv[1])) {
|
||||
process.exit(restoreNodeAbi() ? 0 : 1)
|
||||
}
|
||||
Reference in new issue
Block a user