From 256e93605ae95faebc97fad8b7351d8bb3b87b7b Mon Sep 17 00:00:00 2001 From: digua Date: Mon, 3 Aug 2026 16:13:40 +0800 Subject: [PATCH] refactor(database): remove legacy session FTS --- apps/cli/src/http/routes/web/index.ts | 2 +- apps/cli/src/import/stream-import.ts | 4 +- apps/desktop/main/database/migrations.test.ts | 10 +- apps/desktop/main/database/migrations.ts | 5 +- apps/desktop/main/i18n/locales/en-US.ts | 6 +- apps/desktop/main/i18n/locales/ja-JP.ts | 6 +- apps/desktop/main/i18n/locales/zh-CN.ts | 6 +- apps/desktop/main/i18n/locales/zh-TW.ts | 6 +- apps/desktop/main/worker/query/messages.ts | 2 +- .../main/worker/query/session/aiTools.ts | 2 +- packages/core/src/query/message-queries.ts | 6 +- .../core/src/query/message-query-functions.ts | 2 +- packages/core/src/schema/migrations.ts | 2 +- packages/core/src/schema/tables.ts | 6 +- .../node-runtime/src/database-manager.test.ts | 93 +++++++++++++++++-- packages/node-runtime/src/database-manager.ts | 22 +---- .../src/import/streaming-importer.ts | 2 +- packages/node-runtime/src/index.ts | 3 - .../src/migrations/chat-db-migrations.ts | 67 +++---------- packages/node-runtime/src/migrations/index.ts | 2 +- .../src/nlp/__tests__/fts-tokenizer.test.ts | 53 ----------- .../src/nlp/__tests__/segmenter-cache.test.ts | 2 +- .../node-runtime/src/nlp/fts-tokenizer.ts | 75 --------------- packages/node-runtime/src/nlp/index.ts | 3 - packages/node-runtime/src/nlp/segmenter.ts | 2 +- packages/parser/src/types.ts | 2 +- .../src/definitions/deep-search-messages.ts | 3 +- 27 files changed, 148 insertions(+), 246 deletions(-) delete mode 100644 packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts delete mode 100644 packages/node-runtime/src/nlp/fts-tokenizer.ts diff --git a/apps/cli/src/http/routes/web/index.ts b/apps/cli/src/http/routes/web/index.ts index 4fabf720..62271013 100644 --- a/apps/cli/src/http/routes/web/index.ts +++ b/apps/cli/src/http/routes/web/index.ts @@ -9,7 +9,7 @@ * members – Member management * analytics – Stats and advanced analytics * sql – SQL Lab and plugin query - * sessionIndex – Session index generation + FTS + * sessionIndex – Session index generation * summaries – LLM summary generation * import – File / directory / incremental import + demo * merge – Merge parse / conflicts / execute diff --git a/apps/cli/src/import/stream-import.ts b/apps/cli/src/import/stream-import.ts index 50427516..e2778e33 100644 --- a/apps/cli/src/import/stream-import.ts +++ b/apps/cli/src/import/stream-import.ts @@ -3,7 +3,7 @@ * * Replaces the old buffered-in-memory approach with the same high-performance * streaming pipeline used by Electron (batched transactions, deferred indexes, - * nickname history, FTS, format fallback). + * nickname history, indexes, and format fallback). */ import type { DatabaseManager } from '@openchatlab/node-runtime' @@ -142,7 +142,7 @@ function deleteSessionDatabase(dbManager: DatabaseManager, sessionId: string): v /** * High-performance streaming import: parse a file and write to DB - * with batched transactions, deferred indexes, and FTS. + * with batched transactions and deferred indexes. */ async function streamImportUnlocked( dbManager: DatabaseManager, diff --git a/apps/desktop/main/database/migrations.test.ts b/apps/desktop/main/database/migrations.test.ts index e9bb8689..c55fdc32 100644 --- a/apps/desktop/main/database/migrations.test.ts +++ b/apps/desktop/main/database/migrations.test.ts @@ -86,11 +86,15 @@ test('migrateDatabase writes data directory compatibility meta after segment sch }) test('getPendingMigrationInfos maps each version to its own localized message', () => { + const v4 = getPendingMigrationInfos(3)[0] + assert.equal(v4.version, 4) + assert.doesNotMatch(v4.userMessage, /FTS|full-text|全文/i) + const migrations = getPendingMigrationInfos(6) assert.deepEqual( migrations.map((m) => m.version), - [7, 8] + [7, 8, 9] ) const v7 = migrations[0] @@ -101,4 +105,8 @@ test('getPendingMigrationInfos maps each version to its own localized message', const v8 = migrations[1] assert.match(v8.userMessage, /index|索引|インデックス/i) assert.doesNotMatch(v8.userMessage, /Owner/) + + const v9 = migrations[2] + assert.match(v9.userMessage, /search|index|搜索|搜尋|検索|索引|インデックス/i) + assert.doesNotMatch(v9.userMessage, /Owner/) }) diff --git a/apps/desktop/main/database/migrations.ts b/apps/desktop/main/database/migrations.ts index fa7681bb..ae50368e 100644 --- a/apps/desktop/main/database/migrations.ts +++ b/apps/desktop/main/database/migrations.ts @@ -20,7 +20,6 @@ import { } from '@openchatlab/node-runtime/src/migrations/chat-db-migrations' import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '@openchatlab/node-runtime/src/data-dir-compat' import { t } from '../i18n' -import { tokenizeForFts } from '@openchatlab/node-runtime/src/nlp/fts-tokenizer' export { CURRENT_SCHEMA_VERSION } @@ -83,7 +82,7 @@ export function migrateDatabase( } const beforeVersion = getSchemaVersion(adapter) - const migrations = getChatDbMigrations({ tokenizeForFts }) + const migrations = getChatDbMigrations() const migrated = runMigrations(adapter, migrations, forceRepair) if (!migrated || !options.pathProvider || !options.runtime) return migrated @@ -114,7 +113,7 @@ export function needsMigration(db: Database.Database): boolean { * Get pending migration info for UI display (with i18n). */ export function getPendingMigrationInfos(fromVersion = 0): MigrationInfo[] { - const migrations = getChatDbMigrations({ tokenizeForFts }) + const migrations = getChatDbMigrations() return migrations .filter((m) => m.version > fromVersion) .map((m) => { diff --git a/apps/desktop/main/i18n/locales/en-US.ts b/apps/desktop/main/i18n/locales/en-US.ts index 3bfa84af..de6b991c 100644 --- a/apps/desktop/main/i18n/locales/en-US.ts +++ b/apps/desktop/main/i18n/locales/en-US.ts @@ -46,8 +46,8 @@ export default { migrationV2Message: 'Support member roles, message reply relationships and reply preview', migrationV3Desc: 'Add session index tables (segment, message_context) and session_gap_threshold field', migrationV3Message: 'Support session timeline browsing and AI-enhanced analysis', - migrationV4Desc: 'Create FTS5 full-text search index (message_fts) and build index data', - migrationV4Message: 'Enable full-text search for significantly faster keyword search', + migrationV4Desc: 'Keep the legacy database migration sequence compatible', + migrationV4Message: 'Apply a lightweight compatibility step without rebuilding removed search data', migrationV5Desc: 'Repair legacy member and message fields', migrationV5Message: 'Update legacy database fields for compatibility with the current version', migrationV6Desc: 'Upgrade the session index to the segment schema', @@ -57,6 +57,8 @@ export default { migrationV8Desc: 'Add analysis tool performance indexes', migrationV8Message: 'Add performance indexes for analysis tools to speed up queries without affecting existing data', + migrationV9Desc: 'Remove the obsolete per-session full-text search index', + migrationV9Message: 'Remove an unused derived search index while preserving all chat data', integrityError: 'Database structure is incomplete: missing meta table. Please delete this database file and re-import.', checkFailed: 'Database check failed: {{error}}', diff --git a/apps/desktop/main/i18n/locales/ja-JP.ts b/apps/desktop/main/i18n/locales/ja-JP.ts index 75420d9b..fd3198ba 100644 --- a/apps/desktop/main/i18n/locales/ja-JP.ts +++ b/apps/desktop/main/i18n/locales/ja-JP.ts @@ -47,8 +47,8 @@ export default { migrationV3Desc: 'セッションインデックス関連テーブル(segment、message_context)と session_gap_threshold フィールドを追加', migrationV3Message: 'セッションのタイムライン表示と AI 拡張分析に対応', - migrationV4Desc: 'FTS5 全文検索インデックス(message_fts)を作成しインデックスデータを構築', - migrationV4Message: '全文検索に対応し、キーワード検索速度が大幅に向上', + migrationV4Desc: '旧バージョンのデータベース移行順序を維持', + migrationV4Message: '廃止された検索データを再構築せず、軽量な互換処理のみ実行します', migrationV5Desc: '旧バージョンのメンバーおよびメッセージフィールドを修復', migrationV5Message: '現在のバージョンと互換性を保つため、旧データベースのフィールドを更新します', migrationV6Desc: 'セッションインデックスを segment スキーマに更新', @@ -58,6 +58,8 @@ export default { migrationV8Desc: '分析ツールのパフォーマンスインデックスを追加', migrationV8Message: '分析ツールのクエリを高速化するパフォーマンスインデックスを追加します(既存データには影響しません)', + migrationV9Desc: '廃止された会話単位の全文検索インデックスを削除', + migrationV9Message: '使用されていない派生検索インデックスのみ削除し、チャットデータは保持します', integrityError: 'データベース構造が不完全です:meta テーブルがありません。このデータベースファイルを削除して再インポートすることをお勧めします。', checkFailed: 'データベースチェックに失敗しました: {{error}}', diff --git a/apps/desktop/main/i18n/locales/zh-CN.ts b/apps/desktop/main/i18n/locales/zh-CN.ts index 10e5e331..82db05d3 100644 --- a/apps/desktop/main/i18n/locales/zh-CN.ts +++ b/apps/desktop/main/i18n/locales/zh-CN.ts @@ -45,8 +45,8 @@ export default { migrationV2Message: '支持成员角色、消息回复关系和回复内容预览', migrationV3Desc: '添加会话索引相关表(segment、message_context)和 session_gap_threshold 字段', migrationV3Message: '支持会话时间轴浏览和 AI 增强分析功能', - migrationV4Desc: '创建 FTS5 全文搜索索引(message_fts)并构建索引数据', - migrationV4Message: '支持全文搜索,大幅提升关键词搜索速度', + migrationV4Desc: '保留旧版数据库迁移序列', + migrationV4Message: '执行轻量兼容步骤,不再重建已停用的搜索索引', migrationV5Desc: '修复旧版成员和消息字段', migrationV5Message: '更新旧版数据库字段以兼容当前版本', migrationV6Desc: '将会话索引升级为 segment 结构', @@ -55,6 +55,8 @@ export default { migrationV7Message: '修复会话索引中缺失的消息关联,并保留现有会话和摘要', migrationV8Desc: '添加分析工具性能索引', migrationV8Message: '为分析工具添加性能索引,提升查询速度,不影响现有数据', + migrationV9Desc: '移除已停用的会话全文搜索索引', + migrationV9Message: '清理不再使用的派生搜索索引,聊天数据不会受到影响', integrityError: '数据库结构不完整:缺少 meta 表。建议删除此数据库文件后重新导入。', checkFailed: '数据库检查失败: {{error}}', }, diff --git a/apps/desktop/main/i18n/locales/zh-TW.ts b/apps/desktop/main/i18n/locales/zh-TW.ts index 47e696ea..a0d9f021 100644 --- a/apps/desktop/main/i18n/locales/zh-TW.ts +++ b/apps/desktop/main/i18n/locales/zh-TW.ts @@ -46,8 +46,8 @@ export default { migrationV2Message: '支援成員角色、訊息回覆關係和回覆內容預覽', migrationV3Desc: '新增會話索引相關資料表(segment、message_context)及 session_gap_threshold 欄位', migrationV3Message: '支援會話時間軸瀏覽與 AI 增強分析功能', - migrationV4Desc: '建立 FTS5 全文搜尋索引(message_fts)並建構索引資料', - migrationV4Message: '支援全文搜尋,大幅提升關鍵詞搜尋速度', + migrationV4Desc: '保留舊版資料庫遷移順序', + migrationV4Message: '執行輕量相容步驟,不再重建已停用的搜尋索引', migrationV5Desc: '修復舊版成員和訊息欄位', migrationV5Message: '更新舊版資料庫欄位以相容目前版本', migrationV6Desc: '將會話索引升級為 segment 結構', @@ -56,6 +56,8 @@ export default { migrationV7Message: '修復會話索引中缺少的訊息關聯,並保留現有會話和摘要', migrationV8Desc: '新增分析工具效能索引', migrationV8Message: '為分析工具新增效能索引,提升查詢速度,不影響現有資料', + migrationV9Desc: '移除已停用的會話全文搜尋索引', + migrationV9Message: '清理不再使用的衍生搜尋索引,聊天資料不受影響', integrityError: '資料庫結構不完整:缺少 meta 資料表。建議刪除此資料庫檔案後重新匯入。', checkFailed: '資料庫檢查失敗: {{error}}', }, diff --git a/apps/desktop/main/worker/query/messages.ts b/apps/desktop/main/worker/query/messages.ts index f3132ae5..02d789b4 100644 --- a/apps/desktop/main/worker/query/messages.ts +++ b/apps/desktop/main/worker/query/messages.ts @@ -99,7 +99,7 @@ export async function searchMessages( } /** - * Deep search (always LIKE, no FTS). + * Deep search compatibility alias using the same LIKE implementation. */ export async function deepSearchMessages( sessionId: string, diff --git a/apps/desktop/main/worker/query/session/aiTools.ts b/apps/desktop/main/worker/query/session/aiTools.ts index f113fee2..47992d03 100644 --- a/apps/desktop/main/worker/query/session/aiTools.ts +++ b/apps/desktop/main/worker/query/session/aiTools.ts @@ -1,7 +1,7 @@ /** * AI tool session queries — Electron worker wrappers. * Core search/messages logic lives in @openchatlab/core; - * Electron adds FTS tokenization and DB lifecycle. + * Electron adds database lifecycle management. */ import { getSegmentMessages as coreGetSessionMessages } from '@openchatlab/core' diff --git a/packages/core/src/query/message-queries.ts b/packages/core/src/query/message-queries.ts index f1a46d57..50789762 100644 --- a/packages/core/src/query/message-queries.ts +++ b/packages/core/src/query/message-queries.ts @@ -2,7 +2,7 @@ * 消息查询模块(平台无关) * * 提供消息搜索、分页等基础查询能力。 - * 复杂的 FTS 搜索留在 Electron/Server 层处理(依赖分词器)。 + * 所有运行时统一使用平台无关的 LIKE 查询。 */ import type { TimeFilter } from '@openchatlab/shared-types' @@ -140,7 +140,7 @@ export function queryMessages(db: DatabaseAdapter, options?: QueryMessagesOption /** * 基于 LIKE 的简单关键词搜索 - * 不依赖 FTS 索引,适用于 CLI/MCP 场景 + * 使用 LIKE 子串匹配,适用于 CLI/MCP 场景。 */ export function searchMessagesLike( db: DatabaseAdapter, @@ -202,7 +202,7 @@ export function searchMessagesLike( /** * 多关键词 LIKE 子串搜索(关键词之间 OR,命中任一即返回) * - * 适用于 CLI/MCP/Web 等无 FTS 的场景,行为与 Electron worker 的 + * 适用于 CLI/MCP/Web 等场景,行为与 Electron worker 的 * searchMessagesLikeAsync 一致:支持时间范围、发送者过滤,并排除系统消息。 */ export function searchMessagesByKeywords( diff --git a/packages/core/src/query/message-query-functions.ts b/packages/core/src/query/message-query-functions.ts index ba9cdb26..b4f44d05 100644 --- a/packages/core/src/query/message-query-functions.ts +++ b/packages/core/src/query/message-query-functions.ts @@ -5,7 +5,7 @@ * consume via their respective AsyncSqlExecutor implementations. * * SQL templates, row mapping, and condition builders come from ./message-sql.ts. - * FTS/tokenisation search remains platform-specific — only LIKE-based search is shared here. + * All runtimes share the LIKE-based message search implemented here. */ import type { TimeFilter } from '@openchatlab/shared-types' diff --git a/packages/core/src/schema/migrations.ts b/packages/core/src/schema/migrations.ts index bf751ff6..00504787 100644 --- a/packages/core/src/schema/migrations.ts +++ b/packages/core/src/schema/migrations.ts @@ -2,7 +2,7 @@ * 数据库迁移框架(平台无关) * * 提供 Schema 版本检测和通用迁移执行逻辑。 - * 具体的迁移脚本由各平台运行时注册(因为某些迁移依赖 NLP 分词器等平台特性)。 + * 具体的迁移脚本由运行时注册,迁移本身只依赖 DatabaseAdapter。 */ import type { DatabaseAdapter } from '../interfaces' diff --git a/packages/core/src/schema/tables.ts b/packages/core/src/schema/tables.ts index a8e68027..4122b6cd 100644 --- a/packages/core/src/schema/tables.ts +++ b/packages/core/src/schema/tables.ts @@ -4,11 +4,11 @@ * 所有 CREATE TABLE / INDEX 语句的单一事实来源。 * 新建数据库时使用完整 Schema,现有数据库通过迁移脚本演进。 * - * 当前 Schema 版本:7 + * 当前 Schema 版本:9 */ /** 当前 Schema 版本(最新迁移的版本号) */ -export const CURRENT_SCHEMA_VERSION = 8 +export const CURRENT_SCHEMA_VERSION = 9 /** * Table DDL only (no indexes). Used by bulk-import workflows that defer @@ -92,7 +92,7 @@ export const CHAT_DB_INDEXES = ` ` /** - * Combined tables + indexes DDL (Schema Version 4). + * Combined tables + indexes DDL for the current schema. * For new database creation where deferred indexing is not needed. */ export const CHAT_DB_SCHEMA = CHAT_DB_TABLES + CHAT_DB_INDEXES diff --git a/packages/node-runtime/src/database-manager.test.ts b/packages/node-runtime/src/database-manager.test.ts index ec7fd56f..19f84090 100644 --- a/packages/node-runtime/src/database-manager.test.ts +++ b/packages/node-runtime/src/database-manager.test.ts @@ -114,7 +114,7 @@ test('open migrates legacy member name columns before readonly queries', () => { manager.closeAll() }) -test('open backfills FTS index when migrating legacy sessions', () => { +test('open upgrades v3 sessions without building the removed FTS index', () => { const root = makeTempDir() const dbDir = path.join(root, 'data', 'databases') fs.mkdirSync(dbDir, { recursive: true }) @@ -130,7 +130,7 @@ test('open backfills FTS index when migrating legacy sessions', () => { schema_version INTEGER DEFAULT 3 ); INSERT INTO meta (name, platform, type, imported_at, schema_version) - VALUES ('FTS Legacy Chat', 'qq', 'group', 1000, 3); + VALUES ('Legacy Chat', 'qq', 'group', 1000, 3); CREATE TABLE member ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -158,13 +158,88 @@ test('open backfills FTS index when migrating legacy sessions', () => { const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number } assert.equal(version.schema_version, CURRENT_SCHEMA_VERSION) - const ftsCount = db.prepare('SELECT COUNT(*) as total FROM message_fts').get() as { total: number } - assert.equal(ftsCount.total, 1) + const messages = db.prepare('SELECT type, content FROM message ORDER BY id').all() + assert.deepEqual(messages, [ + { type: 0, content: 'hello searchable history' }, + { type: 1, content: 'image message ignored' }, + ]) + const ftsTable = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get() + assert.equal(ftsTable, undefined) - const searchCount = db - .prepare("SELECT COUNT(*) as total FROM message_fts WHERE content MATCH 'searchable'") - .get() as { total: number } - assert.equal(searchCount.total, 1) + manager.closeAll() +}) + +test('open removes a populated v8 FTS table without changing canonical chat data or raising a new gate', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'v8-with-fts.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(CHAT_DB_SCHEMA) + rawDb.exec(` + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('V8 Chat', 'qq', 'group', 1000, 8); + INSERT INTO member (id, platform_id, account_name) VALUES (1, 'u1', 'Alice'); + INSERT INTO message (id, sender_id, ts, type, content) + VALUES (1, 1, 1000, 0, 'canonical message'); + CREATE VIRTUAL TABLE message_fts USING fts5(content, content='', content_rowid=id); + INSERT INTO message_fts(rowid, content) VALUES (1, 'canonical message'); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { + nativeBinding, + runtime: { version: '0.34.2', kind: 'cli' }, + }) + const db = manager.open('v8-with-fts') + assert.ok(db) + + const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number } + assert.equal(version.schema_version, CURRENT_SCHEMA_VERSION) + assert.deepEqual(db.prepare('SELECT id, content FROM message').get(), { id: 1, content: 'canonical message' }) + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get(), + undefined + ) + + const compatibility = readDataDirCompatibilityMeta(path.join(root, 'data')) + assert.equal(compatibility?.minRuntimeVersion, '0.25.1') + assert.deepEqual(compatibility?.reasons, ['segment-schema']) + + manager.closeAll() +}) + +test('open safely upgrades a v8 session that has no FTS table', () => { + const root = makeTempDir() + const dbDir = path.join(root, 'data', 'databases') + fs.mkdirSync(dbDir, { recursive: true }) + const dbPath = path.join(dbDir, 'v8-without-fts.db') + + const rawDb = new Database(dbPath, { nativeBinding }) + rawDb.exec(CHAT_DB_SCHEMA) + rawDb.exec(` + INSERT INTO meta (name, platform, type, imported_at, schema_version) + VALUES ('V8 Chat', 'qq', 'group', 1000, 8); + INSERT INTO member (id, platform_id, account_name) VALUES (1, 'u1', 'Alice'); + INSERT INTO message (id, sender_id, ts, type, content) + VALUES (1, 1, 1000, 0, 'message without FTS'); + `) + rawDb.close() + + const manager = new DatabaseManager(createPathProvider(root), { nativeBinding, allowMissingRuntimeForTests: true }) + const db = manager.open('v8-without-fts') + assert.ok(db) + + assert.deepEqual(db.prepare('SELECT schema_version, name FROM meta LIMIT 1').get(), { + schema_version: CURRENT_SCHEMA_VERSION, + name: 'V8 Chat', + }) + assert.equal((db.prepare('SELECT COUNT(*) AS count FROM message').get() as { count: number }).count, 1) + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get(), + undefined + ) manager.closeAll() }) @@ -744,8 +819,10 @@ test('openRawSessionDatabase can initialize current chat tables for controlled i const metaTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get() const messageTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message'").get() + const ftsTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get() assert.ok(metaTable) assert.ok(messageTable) + assert.equal(ftsTable, undefined) db.close() }) diff --git a/packages/node-runtime/src/database-manager.ts b/packages/node-runtime/src/database-manager.ts index 67b78019..603b3cb2 100644 --- a/packages/node-runtime/src/database-manager.ts +++ b/packages/node-runtime/src/database-manager.ts @@ -20,28 +20,14 @@ import { import { openBetterSqliteDatabase } from './better-sqlite3-adapter' import { deleteSessionCache } from './cache/session-cache' import { assertDataDirCompatible, type RuntimeIdentity } from './data-dir-compat' -import { - CHAT_DB_COMPATIBILITY_RAISES, - getChatDbMigrations, - raiseChatDbCompatibilityGate, - type MigrationDeps, -} from './migrations' -import { tokenizeForFts } from './nlp/fts-tokenizer' +import { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './migrations' import { getContactsFactsCacheDir } from './services/contacts/paths' import { getGlobalInsightDir, getGlobalInsightFactsCacheDir } from './services/global-insight/paths' import { deleteAnnualSummarySnapshots } from './services/global-insight/snapshot' import { getPeopleRelationshipsFactsCacheDir } from './services/people/relationships/paths' -function createMigrationDeps(overrides?: MigrationDeps): MigrationDeps { - return { - tokenizeForFts, - ...overrides, - } -} - interface DatabaseManagerOptions { nativeBinding?: string - migrationDeps?: MigrationDeps runtime?: RuntimeIdentity allowMissingRuntimeForTests?: boolean } @@ -68,7 +54,6 @@ export function listDatabaseCandidateIds(databaseDir: string): string[] { export class DatabaseManager { private cache = new Map() private nativeBinding?: string - private migrationDeps?: MigrationDeps private runtime: RuntimeIdentity | null constructor( @@ -80,7 +65,6 @@ export class DatabaseManager { } this.nativeBinding = options?.nativeBinding - this.migrationDeps = createMigrationDeps(options?.migrationDeps) this.runtime = options?.runtime ?? null } @@ -128,7 +112,7 @@ export class DatabaseManager { }) try { - const migrations = getChatDbMigrations(this.migrationDeps) + const migrations = getChatDbMigrations() this.runMigrations(adapter, migrations) } finally { adapter.close() @@ -159,7 +143,7 @@ export class DatabaseManager { }) if (coreNeedsMigration(adapter, CURRENT_SCHEMA_VERSION)) { - const migrations = getChatDbMigrations(this.migrationDeps) + const migrations = getChatDbMigrations() this.runMigrations(adapter, migrations) this.assertCompatible() } else { diff --git a/packages/node-runtime/src/import/streaming-importer.ts b/packages/node-runtime/src/import/streaming-importer.ts index a9d8cd26..fce35da7 100644 --- a/packages/node-runtime/src/import/streaming-importer.ts +++ b/packages/node-runtime/src/import/streaming-importer.ts @@ -150,7 +150,7 @@ function elapsedMs(startedAt: number, now: () => number): number { /** * Let the event loop process pending I/O before a long synchronous step. * Without this, progress events written to an SSE socket stay buffered - * until the blocking work (index/FTS build) finishes. + * until the blocking index work finishes. */ const yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve)) diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts index f5fb9c2d..7e6829a3 100644 --- a/packages/node-runtime/src/index.ts +++ b/packages/node-runtime/src/index.ts @@ -178,8 +178,6 @@ export { downloadDict, deleteDict, ensureDefaultDict, - tokenizeForFts, - tokenizeQueryForFts, } from './nlp' // AI 助手/技能解析器 + 对话管理 @@ -386,7 +384,6 @@ export type { OverviewCache, MembersCache, MemberStat } from './cache' // Chat DB migrations export { getChatDbMigrations } from './migrations' -export type { MigrationDeps } from './migrations' // Electron data migration (CLI first-run) + data path verification export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './migrations' diff --git a/packages/node-runtime/src/migrations/chat-db-migrations.ts b/packages/node-runtime/src/migrations/chat-db-migrations.ts index 73a0d6ac..a3e800b2 100644 --- a/packages/node-runtime/src/migrations/chat-db-migrations.ts +++ b/packages/node-runtime/src/migrations/chat-db-migrations.ts @@ -3,7 +3,6 @@ * * Extracted from electron/main/database/migrations.ts. * Migration scripts use only DatabaseAdapter — no Electron or Node-specific APIs. - * Version 4 (FTS backfill) requires a tokenizer injected via MigrationDeps. */ import type { DatabaseAdapter } from '@openchatlab/core' @@ -11,11 +10,6 @@ import type { PathProvider } from '@openchatlab/core' import type { Migration as CoreMigration } from '@openchatlab/core' import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '../data-dir-compat' -export interface MigrationDeps { - /** FTS tokenizer — needed by v4 migration for backfilling the FTS index */ - tokenizeForFts?: (content: string) => string | null -} - export interface ChatDbCompatibilityRaise { migrationVersion: number minRuntimeVersion: string @@ -34,6 +28,9 @@ export const CHAT_DB_COMPATIBILITY_RAISES: ChatDbCompatibilityRaise[] = [ }, ] +// Migration 9 intentionally does not raise the data-directory gate. ChatLab 0.34.2 already treats the +// derived FTS table as optional: searches fall back to LIKE and incremental writers skip missing tables. + export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime: RuntimeIdentity): void { for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) { raiseDataDirMinRuntimeVersion(pathProvider, { @@ -49,10 +46,9 @@ export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime /** * Build the chat DB migration list. * - * @param deps Optional dependencies (tokenizer for FTS backfill) * @returns Array of migrations compatible with core `runMigrations` */ -export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] { +export function getChatDbMigrations(): CoreMigration[] { const hasColumn = (db: DatabaseAdapter, tableName: string, columnName: string): boolean => { const tableInfo = db.pragma(`table_info(${tableName})`) as Array<{ name: string }> return tableInfo.some((col) => col.name === columnName) @@ -144,51 +140,9 @@ export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] { }, { version: 4, - description: 'Add FTS5 full-text search index', - up: (db: DatabaseAdapter) => { - const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get() - if (hasTable) return - - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( - content, - content='', - content_rowid=id - ) - `) - - const tokenize = deps?.tokenizeForFts - if (!tokenize) return - - const BATCH_SIZE = 5000 - const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)') - - const countRow = db - .prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''") - .get() as { total: number } | undefined - - const total = countRow?.total ?? 0 - let offset = 0 - while (offset < total) { - const rows = db - .prepare( - `SELECT id, content FROM message - WHERE type = 0 AND content IS NOT NULL AND content != '' - ORDER BY id ASC LIMIT ? OFFSET ?` - ) - .all(BATCH_SIZE, offset) as Array<{ id: number; content: string }> - - if (rows.length === 0) break - - for (const row of rows) { - const tokens = tokenize(row.content) - if (tokens) { - insertFts.run(row.id, tokens) - } - } - - offset += BATCH_SIZE - } + description: 'Preserve the legacy schema migration sequence', + up: () => { + // Version 4 remains as a compatibility marker. New runtimes no longer build the removed derived FTS table. }, }, { @@ -411,5 +365,12 @@ export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] { `) }, }, + { + version: 9, + description: 'Remove the obsolete per-session full-text index', + up: (db: DatabaseAdapter) => { + db.exec('DROP TABLE IF EXISTS message_fts') + }, + }, ] } diff --git a/packages/node-runtime/src/migrations/index.ts b/packages/node-runtime/src/migrations/index.ts index bbf939bd..774af103 100644 --- a/packages/node-runtime/src/migrations/index.ts +++ b/packages/node-runtime/src/migrations/index.ts @@ -1,5 +1,5 @@ export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './chat-db-migrations' -export type { ChatDbCompatibilityRaise, MigrationDeps } from './chat-db-migrations' +export type { ChatDbCompatibilityRaise } from './chat-db-migrations' export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './electron-data-migration' export type { ElectronMigrationResult } from './electron-data-migration' diff --git a/packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts b/packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts deleted file mode 100644 index 076ac06b..00000000 --- a/packages/node-runtime/src/nlp/__tests__/fts-tokenizer.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import assert from 'node:assert/strict' -import { describe, it } from 'node:test' -import { tokenizeForFts, tokenizeQueryForFts } from '../fts-tokenizer' - -describe('tokenizeForFts', () => { - it('returns empty string for null/undefined/empty', () => { - assert.equal(tokenizeForFts(null), '') - assert.equal(tokenizeForFts(undefined), '') - assert.equal(tokenizeForFts(''), '') - assert.equal(tokenizeForFts(' '), '') - }) - - it('tokenizes text and lowercases tokens', () => { - const result = tokenizeForFts('Hello World') - assert.ok(result.length > 0) - assert.equal(result, result.toLowerCase()) - }) - - it('produces non-empty output for Chinese text', () => { - const result = tokenizeForFts('今天天气很好') - assert.ok(result.length > 0) - }) - - it('handles mixed CJK and Latin text', () => { - const result = tokenizeForFts('Hello 你好 World') - assert.ok(result.includes('hello')) - }) -}) - -describe('tokenizeQueryForFts', () => { - it('returns empty string for empty array', () => { - assert.equal(tokenizeQueryForFts([]), '') - }) - - it('returns empty string for array of whitespace', () => { - assert.equal(tokenizeQueryForFts([' ', '']), '') - }) - - it('wraps single simple keyword in quotes', () => { - const result = tokenizeQueryForFts(['hello']) - assert.ok(result.includes('"hello"')) - }) - - it('joins multiple keywords with OR', () => { - const result = tokenizeQueryForFts(['hello', 'world']) - assert.ok(result.includes('OR')) - }) - - it('handles Chinese keywords', () => { - const result = tokenizeQueryForFts(['今天开心']) - assert.ok(result.length > 0) - }) -}) diff --git a/packages/node-runtime/src/nlp/__tests__/segmenter-cache.test.ts b/packages/node-runtime/src/nlp/__tests__/segmenter-cache.test.ts index fd5e8c49..1add5b1d 100644 --- a/packages/node-runtime/src/nlp/__tests__/segmenter-cache.test.ts +++ b/packages/node-runtime/src/nlp/__tests__/segmenter-cache.test.ts @@ -3,7 +3,7 @@ * * Bug history: when no custom dict existed on disk, every getJieba() call * invalidated the cache and rebuilt a Jieba instance, making hot paths - * (FTS indexing tokenizes every message) rebuild jieba per call and slowing + * (bulk NLP analysis tokenizes many messages) rebuild jieba per call and slowing * imports by seconds. The cache must only invalidate when the on-disk dict * state actually changes (dict added or removed). */ diff --git a/packages/node-runtime/src/nlp/fts-tokenizer.ts b/packages/node-runtime/src/nlp/fts-tokenizer.ts deleted file mode 100644 index 9775240d..00000000 --- a/packages/node-runtime/src/nlp/fts-tokenizer.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * FTS5 tokenizer — shared implementation. - * - * Unlike NLP word-frequency analysis, FTS tokenization keeps all tokens - * (no POS filtering, no stopwords) to maximize recall in search scenarios. - * - * Uses jieba for Chinese (naturally handles mixed CJK/Latin text), - * falls back to whitespace split when jieba is unavailable. - */ - -import { getJieba } from './segmenter' - -export function tokenizeForFts(text: string | null | undefined): string { - if (!text || text.trim().length === 0) return '' - - try { - const jieba = getJieba() - const tokens = jieba.cut(text, false) - return tokens - .map((t) => t.trim().toLowerCase()) - .filter((t) => t.length > 0) - .join(' ') - } catch { - return fallbackTokenize(text) - } -} - -function fallbackTokenize(text: string): string { - return text - .toLowerCase() - .split(/\s+/) - .filter((t) => t.length > 0) - .join(' ') -} - -function escapeToken(token: string): string { - return `"${token.replace(/"/g, '""')}"` -} - -/** - * Convert a list of user search keywords into an FTS5 MATCH expression. - * - * - Single keyword: tokens joined with AND (all must appear) - * - Multiple keywords: groups joined with OR (any match) - */ -export function tokenizeQueryForFts(keywords: string[]): string { - if (keywords.length === 0) return '' - - const groups = keywords - .map((kw) => { - const trimmed = kw.trim() - if (!trimmed) return '' - - try { - const jieba = getJieba() - const tokens = jieba - .cut(trimmed, false) - .map((t) => t.trim().toLowerCase()) - .filter((t) => t.length > 0) - - if (tokens.length === 0) return '' - if (tokens.length === 1) return escapeToken(tokens[0]) - return tokens.map(escapeToken).join(' ') - } catch { - const simple = trimmed.toLowerCase().trim() - return simple ? escapeToken(simple) : '' - } - }) - .filter((g) => g.length > 0) - - if (groups.length === 0) return '' - if (groups.length === 1) return groups[0] - - return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ') -} diff --git a/packages/node-runtime/src/nlp/index.ts b/packages/node-runtime/src/nlp/index.ts index 38a2de2f..d40ff0fd 100644 --- a/packages/node-runtime/src/nlp/index.ts +++ b/packages/node-runtime/src/nlp/index.ts @@ -20,9 +20,6 @@ export { // 词频统计 export { computeWordFrequency, segmentText } from './word-frequency' -// FTS tokenizer -export { tokenizeForFts, tokenizeQueryForFts } from './fts-tokenizer' - // 词库管理 export { isDictDownloaded, diff --git a/packages/node-runtime/src/nlp/segmenter.ts b/packages/node-runtime/src/nlp/segmenter.ts index c8fb0b77..5fba9143 100644 --- a/packages/node-runtime/src/nlp/segmenter.ts +++ b/packages/node-runtime/src/nlp/segmenter.ts @@ -69,7 +69,7 @@ export function getJieba(dictType: DictType = 'default'): JiebaInstance { if (cached) { // Only invalidate when the on-disk dict state changed since the instance // was built (dict added or removed). Instances built without a disk dict - // must stay cached, otherwise hot paths (e.g. FTS indexing) would rebuild + // must stay cached, otherwise bulk NLP hot paths would rebuild // a Jieba instance per call. if (cached.fromDiskDict === existsDictOnDisk(effectiveType)) return cached.instance jiebaInstances.delete(effectiveType) diff --git a/packages/parser/src/types.ts b/packages/parser/src/types.ts index 192f2586..444c544c 100644 --- a/packages/parser/src/types.ts +++ b/packages/parser/src/types.ts @@ -26,7 +26,7 @@ export interface ParsedMeta { export interface ParseProgress { /** 阶段 */ // 导入流程会复用解析进度结构,因此这里补充导入阶段枚举: - // saving = 消息批量写库,indexing = 建索引/FTS/会话分段等后处理。 + // saving = 消息批量写库,indexing = 建普通索引/会话分段等后处理。 stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error' /** 已读取字节数 */ bytesRead: number diff --git a/packages/tools/src/definitions/deep-search-messages.ts b/packages/tools/src/definitions/deep-search-messages.ts index 14bd1643..9895bcd8 100644 --- a/packages/tools/src/definitions/deep-search-messages.ts +++ b/packages/tools/src/definitions/deep-search-messages.ts @@ -58,8 +58,7 @@ async function handler(params: Record, context: ToolExecutionCo export const deepSearchMessagesTool: ToolDefinition = { name: 'deep_search_messages', - description: - '深度搜索消息(LIKE 子串匹配,速度较慢但不会遗漏),适用于全文搜索精确匹配或 FTS 搜索无结果时的补充搜索。', + description: '深度搜索消息(LIKE 子串匹配),适用于需要精确文本片段匹配的场景。', inputSchema, handler, category: 'core',