refactor(database): remove legacy session FTS

This commit is contained in:
digua committed 2026-08-03 17:28:19 +08:00
1 parent ef4f8de74d
commit 256e93605a
27 files changed
+148 -246

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@
* members – Member management
* analytics – Stats and advanced analytics
* sql – SQL Lab and plugin query
* sessionIndex – Session index generation + FTS
* sessionIndex – Session index generation
* summaries – LLM summary generation
* import – File / directory / incremental import + demo
* merge – Merge parse / conflicts / execute
+2 -2
View File
@@ -3,7 +3,7 @@
*
* Replaces the old buffered-in-memory approach with the same high-performance
* streaming pipeline used by Electron (batched transactions, deferred indexes,
* nickname history, FTS, format fallback).
* nickname history, indexes, and format fallback).
*/
import type { DatabaseManager } from '@openchatlab/node-runtime'
@@ -142,7 +142,7 @@ function deleteSessionDatabase(dbManager: DatabaseManager, sessionId: string): v
/**
* High-performance streaming import: parse a file and write to DB
* with batched transactions, deferred indexes, and FTS.
* with batched transactions and deferred indexes.
*/
async function streamImportUnlocked(
dbManager: DatabaseManager,
@@ -86,11 +86,15 @@ test('migrateDatabase writes data directory compatibility meta after segment sch
})
test('getPendingMigrationInfos maps each version to its own localized message', () => {
const v4 = getPendingMigrationInfos(3)[0]
assert.equal(v4.version, 4)
assert.doesNotMatch(v4.userMessage, /FTS|full-text|全文/i)
const migrations = getPendingMigrationInfos(6)
assert.deepEqual(
migrations.map((m) => m.version),
[7, 8]
[7, 8, 9]
)
const v7 = migrations[0]
@@ -101,4 +105,8 @@ test('getPendingMigrationInfos maps each version to its own localized message',
const v8 = migrations[1]
assert.match(v8.userMessage, /index|索引|インデックス/i)
assert.doesNotMatch(v8.userMessage, /Owner/)
const v9 = migrations[2]
assert.match(v9.userMessage, /search|index|搜索|搜尋|検索|索引|インデックス/i)
assert.doesNotMatch(v9.userMessage, /Owner/)
})
+2 -3
View File
@@ -20,7 +20,6 @@ import {
} from '@openchatlab/node-runtime/src/migrations/chat-db-migrations'
import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '@openchatlab/node-runtime/src/data-dir-compat'
import { t } from '../i18n'
import { tokenizeForFts } from '@openchatlab/node-runtime/src/nlp/fts-tokenizer'
export { CURRENT_SCHEMA_VERSION }
@@ -83,7 +82,7 @@ export function migrateDatabase(
}
const beforeVersion = getSchemaVersion(adapter)
const migrations = getChatDbMigrations({ tokenizeForFts })
const migrations = getChatDbMigrations()
const migrated = runMigrations(adapter, migrations, forceRepair)
if (!migrated || !options.pathProvider || !options.runtime) return migrated
@@ -114,7 +113,7 @@ export function needsMigration(db: Database.Database): boolean {
* Get pending migration info for UI display (with i18n).
*/
export function getPendingMigrationInfos(fromVersion = 0): MigrationInfo[] {
const migrations = getChatDbMigrations({ tokenizeForFts })
const migrations = getChatDbMigrations()
return migrations
.filter((m) => m.version > fromVersion)
.map((m) => {
+4 -2
View File
@@ -46,8 +46,8 @@ export default {
migrationV2Message: 'Support member roles, message reply relationships and reply preview',
migrationV3Desc: 'Add session index tables (segment, message_context) and session_gap_threshold field',
migrationV3Message: 'Support session timeline browsing and AI-enhanced analysis',
migrationV4Desc: 'Create FTS5 full-text search index (message_fts) and build index data',
migrationV4Message: 'Enable full-text search for significantly faster keyword search',
migrationV4Desc: 'Keep the legacy database migration sequence compatible',
migrationV4Message: 'Apply a lightweight compatibility step without rebuilding removed search data',
migrationV5Desc: 'Repair legacy member and message fields',
migrationV5Message: 'Update legacy database fields for compatibility with the current version',
migrationV6Desc: 'Upgrade the session index to the segment schema',
@@ -57,6 +57,8 @@ export default {
migrationV8Desc: 'Add analysis tool performance indexes',
migrationV8Message:
'Add performance indexes for analysis tools to speed up queries without affecting existing data',
migrationV9Desc: 'Remove the obsolete per-session full-text search index',
migrationV9Message: 'Remove an unused derived search index while preserving all chat data',
integrityError:
'Database structure is incomplete: missing meta table. Please delete this database file and re-import.',
checkFailed: 'Database check failed: {{error}}',
+4 -2
View File
@@ -47,8 +47,8 @@ export default {
migrationV3Desc:
'セッションインデックス関連テーブル(segment、message_context)と session_gap_threshold フィールドを追加',
migrationV3Message: 'セッションのタイムライン表示と AI 拡張分析に対応',
migrationV4Desc: 'FTS5 全文検索インデックス(message_fts)を作成しインデックスデータを構築',
migrationV4Message: '全文検索に対応し、キーワード検索速度が大幅に向上',
migrationV4Desc: '旧バージョンのデータベース移行順序を維持',
migrationV4Message: '廃止された検索データを再構築せず、軽量な互換処理のみ実行します',
migrationV5Desc: '旧バージョンのメンバーおよびメッセージフィールドを修復',
migrationV5Message: '現在のバージョンと互換性を保つため、旧データベースのフィールドを更新します',
migrationV6Desc: 'セッションインデックスを segment スキーマに更新',
@@ -58,6 +58,8 @@ export default {
migrationV8Desc: '分析ツールのパフォーマンスインデックスを追加',
migrationV8Message:
'分析ツールのクエリを高速化するパフォーマンスインデックスを追加します(既存データには影響しません)',
migrationV9Desc: '廃止された会話単位の全文検索インデックスを削除',
migrationV9Message: '使用されていない派生検索インデックスのみ削除し、チャットデータは保持します',
integrityError:
'データベース構造が不完全です:meta テーブルがありません。このデータベースファイルを削除して再インポートすることをお勧めします。',
checkFailed: 'データベースチェックに失敗しました: {{error}}',
+4 -2
View File
@@ -45,8 +45,8 @@ export default {
migrationV2Message: '支持成员角色、消息回复关系和回复内容预览',
migrationV3Desc: '添加会话索引相关表(segment、message_context)和 session_gap_threshold 字段',
migrationV3Message: '支持会话时间轴浏览和 AI 增强分析功能',
migrationV4Desc: '创建 FTS5 全文搜索索引(message_fts)并构建索引数据',
migrationV4Message: '支持全文搜索,大幅提升关键词搜索速度',
migrationV4Desc: '保留旧版数据库迁移序列',
migrationV4Message: '执行轻量兼容步骤,不再重建已停用的搜索索引',
migrationV5Desc: '修复旧版成员和消息字段',
migrationV5Message: '更新旧版数据库字段以兼容当前版本',
migrationV6Desc: '将会话索引升级为 segment 结构',
@@ -55,6 +55,8 @@ export default {
migrationV7Message: '修复会话索引中缺失的消息关联,并保留现有会话和摘要',
migrationV8Desc: '添加分析工具性能索引',
migrationV8Message: '为分析工具添加性能索引,提升查询速度,不影响现有数据',
migrationV9Desc: '移除已停用的会话全文搜索索引',
migrationV9Message: '清理不再使用的派生搜索索引,聊天数据不会受到影响',
integrityError: '数据库结构不完整:缺少 meta 表。建议删除此数据库文件后重新导入。',
checkFailed: '数据库检查失败: {{error}}',
},
+4 -2
View File
@@ -46,8 +46,8 @@ export default {
migrationV2Message: '支援成員角色、訊息回覆關係和回覆內容預覽',
migrationV3Desc: '新增會話索引相關資料表(segment、message_context)及 session_gap_threshold 欄位',
migrationV3Message: '支援會話時間軸瀏覽與 AI 增強分析功能',
migrationV4Desc: '建立 FTS5 全文搜尋索引(message_fts)並建構索引資料',
migrationV4Message: '支援全文搜尋,大幅提升關鍵詞搜尋速度',
migrationV4Desc: '保留舊版資料庫遷移順序',
migrationV4Message: '執行輕量相容步驟,不再重建已停用的搜尋索引',
migrationV5Desc: '修復舊版成員和訊息欄位',
migrationV5Message: '更新舊版資料庫欄位以相容目前版本',
migrationV6Desc: '將會話索引升級為 segment 結構',
@@ -56,6 +56,8 @@ export default {
migrationV7Message: '修復會話索引中缺少的訊息關聯,並保留現有會話和摘要',
migrationV8Desc: '新增分析工具效能索引',
migrationV8Message: '為分析工具新增效能索引,提升查詢速度,不影響現有資料',
migrationV9Desc: '移除已停用的會話全文搜尋索引',
migrationV9Message: '清理不再使用的衍生搜尋索引,聊天資料不受影響',
integrityError: '資料庫結構不完整:缺少 meta 資料表。建議刪除此資料庫檔案後重新匯入。',
checkFailed: '資料庫檢查失敗: {{error}}',
},
+1 -1
View File
@@ -99,7 +99,7 @@ export async function searchMessages(
}
/**
* Deep search (always LIKE, no FTS).
* Deep search compatibility alias using the same LIKE implementation.
*/
export async function deepSearchMessages(
sessionId: string,
@@ -1,7 +1,7 @@
/**
* AI tool session queries — Electron worker wrappers.
* Core search/messages logic lives in @openchatlab/core;
* Electron adds FTS tokenization and DB lifecycle.
* Electron adds database lifecycle management.
*/
import { getSegmentMessages as coreGetSessionMessages } from '@openchatlab/core'
+3 -3
View File
@@ -2,7 +2,7 @@
* 消息查询模块(平台无关)
*
* 提供消息搜索、分页等基础查询能力。
* 复杂的 FTS 搜索留在 Electron/Server 层处理(依赖分词器)。
* 所有运行时统一使用平台无关的 LIKE 查询。
*/
import type { TimeFilter } from '@openchatlab/shared-types'
@@ -140,7 +140,7 @@ export function queryMessages(db: DatabaseAdapter, options?: QueryMessagesOption
/**
* 基于 LIKE 的简单关键词搜索
* 不依赖 FTS 索引,适用于 CLI/MCP 场景
* 使用 LIKE 子串匹配,适用于 CLI/MCP 场景。
*/
export function searchMessagesLike(
db: DatabaseAdapter,
@@ -202,7 +202,7 @@ export function searchMessagesLike(
/**
* 多关键词 LIKE 子串搜索(关键词之间 OR,命中任一即返回)
*
* 适用于 CLI/MCP/Web 等无 FTS 的场景,行为与 Electron worker 的
* 适用于 CLI/MCP/Web 等场景,行为与 Electron worker 的
* searchMessagesLikeAsync 一致:支持时间范围、发送者过滤,并排除系统消息。
*/
export function searchMessagesByKeywords(
@@ -5,7 +5,7 @@
* consume via their respective AsyncSqlExecutor implementations.
*
* SQL templates, row mapping, and condition builders come from ./message-sql.ts.
* FTS/tokenisation search remains platform-specific — only LIKE-based search is shared here.
* All runtimes share the LIKE-based message search implemented here.
*/
import type { TimeFilter } from '@openchatlab/shared-types'
+1 -1
View File
@@ -2,7 +2,7 @@
* 数据库迁移框架(平台无关)
*
* 提供 Schema 版本检测和通用迁移执行逻辑。
* 具体的迁移脚本由各平台运行时注册(因为某些迁移依赖 NLP 分词器等平台特性)。
* 具体的迁移脚本由运行时注册,迁移本身只依赖 DatabaseAdapter。
*/
import type { DatabaseAdapter } from '../interfaces'
+3 -3
View File
@@ -4,11 +4,11 @@
* 所有 CREATE TABLE / INDEX 语句的单一事实来源。
* 新建数据库时使用完整 Schema,现有数据库通过迁移脚本演进。
*
* 当前 Schema 版本:7
* 当前 Schema 版本:9
*/
/** 当前 Schema 版本(最新迁移的版本号) */
export const CURRENT_SCHEMA_VERSION = 8
export const CURRENT_SCHEMA_VERSION = 9
/**
* Table DDL only (no indexes). Used by bulk-import workflows that defer
@@ -92,7 +92,7 @@ export const CHAT_DB_INDEXES = `
`
/**
* Combined tables + indexes DDL (Schema Version 4).
* Combined tables + indexes DDL for the current schema.
* For new database creation where deferred indexing is not needed.
*/
export const CHAT_DB_SCHEMA = CHAT_DB_TABLES + CHAT_DB_INDEXES
@@ -114,7 +114,7 @@ test('open migrates legacy member name columns before readonly queries', () => {
manager.closeAll()
})
test('open backfills FTS index when migrating legacy sessions', () => {
test('open upgrades v3 sessions without building the removed FTS index', () => {
const root = makeTempDir()
const dbDir = path.join(root, 'data', 'databases')
fs.mkdirSync(dbDir, { recursive: true })
@@ -130,7 +130,7 @@ test('open backfills FTS index when migrating legacy sessions', () => {
schema_version INTEGER DEFAULT 3
);
INSERT INTO meta (name, platform, type, imported_at, schema_version)
VALUES ('FTS Legacy Chat', 'qq', 'group', 1000, 3);
VALUES ('Legacy Chat', 'qq', 'group', 1000, 3);
CREATE TABLE member (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -158,13 +158,88 @@ test('open backfills FTS index when migrating legacy sessions', () => {
const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number }
assert.equal(version.schema_version, CURRENT_SCHEMA_VERSION)
const ftsCount = db.prepare('SELECT COUNT(*) as total FROM message_fts').get() as { total: number }
assert.equal(ftsCount.total, 1)
const messages = db.prepare('SELECT type, content FROM message ORDER BY id').all()
assert.deepEqual(messages, [
{ type: 0, content: 'hello searchable history' },
{ type: 1, content: 'image message ignored' },
])
const ftsTable = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get()
assert.equal(ftsTable, undefined)
const searchCount = db
.prepare("SELECT COUNT(*) as total FROM message_fts WHERE content MATCH 'searchable'")
.get() as { total: number }
assert.equal(searchCount.total, 1)
manager.closeAll()
})
test('open removes a populated v8 FTS table without changing canonical chat data or raising a new gate', () => {
const root = makeTempDir()
const dbDir = path.join(root, 'data', 'databases')
fs.mkdirSync(dbDir, { recursive: true })
const dbPath = path.join(dbDir, 'v8-with-fts.db')
const rawDb = new Database(dbPath, { nativeBinding })
rawDb.exec(CHAT_DB_SCHEMA)
rawDb.exec(`
INSERT INTO meta (name, platform, type, imported_at, schema_version)
VALUES ('V8 Chat', 'qq', 'group', 1000, 8);
INSERT INTO member (id, platform_id, account_name) VALUES (1, 'u1', 'Alice');
INSERT INTO message (id, sender_id, ts, type, content)
VALUES (1, 1, 1000, 0, 'canonical message');
CREATE VIRTUAL TABLE message_fts USING fts5(content, content='', content_rowid=id);
INSERT INTO message_fts(rowid, content) VALUES (1, 'canonical message');
`)
rawDb.close()
const manager = new DatabaseManager(createPathProvider(root), {
nativeBinding,
runtime: { version: '0.34.2', kind: 'cli' },
})
const db = manager.open('v8-with-fts')
assert.ok(db)
const version = db.prepare('SELECT schema_version FROM meta LIMIT 1').get() as { schema_version: number }
assert.equal(version.schema_version, CURRENT_SCHEMA_VERSION)
assert.deepEqual(db.prepare('SELECT id, content FROM message').get(), { id: 1, content: 'canonical message' })
assert.equal(
db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get(),
undefined
)
const compatibility = readDataDirCompatibilityMeta(path.join(root, 'data'))
assert.equal(compatibility?.minRuntimeVersion, '0.25.1')
assert.deepEqual(compatibility?.reasons, ['segment-schema'])
manager.closeAll()
})
test('open safely upgrades a v8 session that has no FTS table', () => {
const root = makeTempDir()
const dbDir = path.join(root, 'data', 'databases')
fs.mkdirSync(dbDir, { recursive: true })
const dbPath = path.join(dbDir, 'v8-without-fts.db')
const rawDb = new Database(dbPath, { nativeBinding })
rawDb.exec(CHAT_DB_SCHEMA)
rawDb.exec(`
INSERT INTO meta (name, platform, type, imported_at, schema_version)
VALUES ('V8 Chat', 'qq', 'group', 1000, 8);
INSERT INTO member (id, platform_id, account_name) VALUES (1, 'u1', 'Alice');
INSERT INTO message (id, sender_id, ts, type, content)
VALUES (1, 1, 1000, 0, 'message without FTS');
`)
rawDb.close()
const manager = new DatabaseManager(createPathProvider(root), { nativeBinding, allowMissingRuntimeForTests: true })
const db = manager.open('v8-without-fts')
assert.ok(db)
assert.deepEqual(db.prepare('SELECT schema_version, name FROM meta LIMIT 1').get(), {
schema_version: CURRENT_SCHEMA_VERSION,
name: 'V8 Chat',
})
assert.equal((db.prepare('SELECT COUNT(*) AS count FROM message').get() as { count: number }).count, 1)
assert.equal(
db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get(),
undefined
)
manager.closeAll()
})
@@ -744,8 +819,10 @@ test('openRawSessionDatabase can initialize current chat tables for controlled i
const metaTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get()
const messageTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message'").get()
const ftsTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'message_fts'").get()
assert.ok(metaTable)
assert.ok(messageTable)
assert.equal(ftsTable, undefined)
db.close()
})
+3 -19
View File
@@ -20,28 +20,14 @@ import {
import { openBetterSqliteDatabase } from './better-sqlite3-adapter'
import { deleteSessionCache } from './cache/session-cache'
import { assertDataDirCompatible, type RuntimeIdentity } from './data-dir-compat'
import {
CHAT_DB_COMPATIBILITY_RAISES,
getChatDbMigrations,
raiseChatDbCompatibilityGate,
type MigrationDeps,
} from './migrations'
import { tokenizeForFts } from './nlp/fts-tokenizer'
import { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './migrations'
import { getContactsFactsCacheDir } from './services/contacts/paths'
import { getGlobalInsightDir, getGlobalInsightFactsCacheDir } from './services/global-insight/paths'
import { deleteAnnualSummarySnapshots } from './services/global-insight/snapshot'
import { getPeopleRelationshipsFactsCacheDir } from './services/people/relationships/paths'
function createMigrationDeps(overrides?: MigrationDeps): MigrationDeps {
return {
tokenizeForFts,
...overrides,
}
}
interface DatabaseManagerOptions {
nativeBinding?: string
migrationDeps?: MigrationDeps
runtime?: RuntimeIdentity
allowMissingRuntimeForTests?: boolean
}
@@ -68,7 +54,6 @@ export function listDatabaseCandidateIds(databaseDir: string): string[] {
export class DatabaseManager {
private cache = new Map<string, DatabaseAdapter>()
private nativeBinding?: string
private migrationDeps?: MigrationDeps
private runtime: RuntimeIdentity | null
constructor(
@@ -80,7 +65,6 @@ export class DatabaseManager {
}
this.nativeBinding = options?.nativeBinding
this.migrationDeps = createMigrationDeps(options?.migrationDeps)
this.runtime = options?.runtime ?? null
}
@@ -128,7 +112,7 @@ export class DatabaseManager {
})
try {
const migrations = getChatDbMigrations(this.migrationDeps)
const migrations = getChatDbMigrations()
this.runMigrations(adapter, migrations)
} finally {
adapter.close()
@@ -159,7 +143,7 @@ export class DatabaseManager {
})
if (coreNeedsMigration(adapter, CURRENT_SCHEMA_VERSION)) {
const migrations = getChatDbMigrations(this.migrationDeps)
const migrations = getChatDbMigrations()
this.runMigrations(adapter, migrations)
this.assertCompatible()
} else {
@@ -150,7 +150,7 @@ function elapsedMs(startedAt: number, now: () => number): number {
/**
* Let the event loop process pending I/O before a long synchronous step.
* Without this, progress events written to an SSE socket stay buffered
* until the blocking work (index/FTS build) finishes.
* until the blocking index work finishes.
*/
const yieldToEventLoop = () => new Promise<void>((resolve) => setImmediate(resolve))
-3
View File
@@ -178,8 +178,6 @@ export {
downloadDict,
deleteDict,
ensureDefaultDict,
tokenizeForFts,
tokenizeQueryForFts,
} from './nlp'
// AI 助手/技能解析器 + 对话管理
@@ -386,7 +384,6 @@ export type { OverviewCache, MembersCache, MemberStat } from './cache'
// Chat DB migrations
export { getChatDbMigrations } from './migrations'
export type { MigrationDeps } from './migrations'
// Electron data migration (CLI first-run) + data path verification
export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './migrations'
@@ -3,7 +3,6 @@
*
* Extracted from electron/main/database/migrations.ts.
* Migration scripts use only DatabaseAdapter — no Electron or Node-specific APIs.
* Version 4 (FTS backfill) requires a tokenizer injected via MigrationDeps.
*/
import type { DatabaseAdapter } from '@openchatlab/core'
@@ -11,11 +10,6 @@ import type { PathProvider } from '@openchatlab/core'
import type { Migration as CoreMigration } from '@openchatlab/core'
import { raiseDataDirMinRuntimeVersion, type RuntimeIdentity } from '../data-dir-compat'
export interface MigrationDeps {
/** FTS tokenizer — needed by v4 migration for backfilling the FTS index */
tokenizeForFts?: (content: string) => string | null
}
export interface ChatDbCompatibilityRaise {
migrationVersion: number
minRuntimeVersion: string
@@ -34,6 +28,9 @@ export const CHAT_DB_COMPATIBILITY_RAISES: ChatDbCompatibilityRaise[] = [
},
]
// Migration 9 intentionally does not raise the data-directory gate. ChatLab 0.34.2 already treats the
// derived FTS table as optional: searches fall back to LIKE and incremental writers skip missing tables.
export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime: RuntimeIdentity): void {
for (const compatibilityRaise of CHAT_DB_COMPATIBILITY_RAISES) {
raiseDataDirMinRuntimeVersion(pathProvider, {
@@ -49,10 +46,9 @@ export function raiseChatDbCompatibilityGate(pathProvider: PathProvider, runtime
/**
* Build the chat DB migration list.
*
* @param deps Optional dependencies (tokenizer for FTS backfill)
* @returns Array of migrations compatible with core `runMigrations`
*/
export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] {
export function getChatDbMigrations(): CoreMigration[] {
const hasColumn = (db: DatabaseAdapter, tableName: string, columnName: string): boolean => {
const tableInfo = db.pragma(`table_info(${tableName})`) as Array<{ name: string }>
return tableInfo.some((col) => col.name === columnName)
@@ -144,51 +140,9 @@ export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] {
},
{
version: 4,
description: 'Add FTS5 full-text search index',
up: (db: DatabaseAdapter) => {
const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='message_fts'").get()
if (hasTable) return
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5(
content,
content='',
content_rowid=id
)
`)
const tokenize = deps?.tokenizeForFts
if (!tokenize) return
const BATCH_SIZE = 5000
const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)')
const countRow = db
.prepare("SELECT COUNT(*) as total FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''")
.get() as { total: number } | undefined
const total = countRow?.total ?? 0
let offset = 0
while (offset < total) {
const rows = db
.prepare(
`SELECT id, content FROM message
WHERE type = 0 AND content IS NOT NULL AND content != ''
ORDER BY id ASC LIMIT ? OFFSET ?`
)
.all(BATCH_SIZE, offset) as Array<{ id: number; content: string }>
if (rows.length === 0) break
for (const row of rows) {
const tokens = tokenize(row.content)
if (tokens) {
insertFts.run(row.id, tokens)
}
}
offset += BATCH_SIZE
}
description: 'Preserve the legacy schema migration sequence',
up: () => {
// Version 4 remains as a compatibility marker. New runtimes no longer build the removed derived FTS table.
},
},
{
@@ -411,5 +365,12 @@ export function getChatDbMigrations(deps?: MigrationDeps): CoreMigration[] {
`)
},
},
{
version: 9,
description: 'Remove the obsolete per-session full-text index',
up: (db: DatabaseAdapter) => {
db.exec('DROP TABLE IF EXISTS message_fts')
},
},
]
}
@@ -1,5 +1,5 @@
export { CHAT_DB_COMPATIBILITY_RAISES, getChatDbMigrations, raiseChatDbCompatibilityGate } from './chat-db-migrations'
export type { ChatDbCompatibilityRaise, MigrationDeps } from './chat-db-migrations'
export type { ChatDbCompatibilityRaise } from './chat-db-migrations'
export { migrateFromElectronIfNeeded, verifyCliDataPath, wasElectronUsed } from './electron-data-migration'
export type { ElectronMigrationResult } from './electron-data-migration'
@@ -1,53 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { tokenizeForFts, tokenizeQueryForFts } from '../fts-tokenizer'
describe('tokenizeForFts', () => {
it('returns empty string for null/undefined/empty', () => {
assert.equal(tokenizeForFts(null), '')
assert.equal(tokenizeForFts(undefined), '')
assert.equal(tokenizeForFts(''), '')
assert.equal(tokenizeForFts(' '), '')
})
it('tokenizes text and lowercases tokens', () => {
const result = tokenizeForFts('Hello World')
assert.ok(result.length > 0)
assert.equal(result, result.toLowerCase())
})
it('produces non-empty output for Chinese text', () => {
const result = tokenizeForFts('今天天气很好')
assert.ok(result.length > 0)
})
it('handles mixed CJK and Latin text', () => {
const result = tokenizeForFts('Hello 你好 World')
assert.ok(result.includes('hello'))
})
})
describe('tokenizeQueryForFts', () => {
it('returns empty string for empty array', () => {
assert.equal(tokenizeQueryForFts([]), '')
})
it('returns empty string for array of whitespace', () => {
assert.equal(tokenizeQueryForFts([' ', '']), '')
})
it('wraps single simple keyword in quotes', () => {
const result = tokenizeQueryForFts(['hello'])
assert.ok(result.includes('"hello"'))
})
it('joins multiple keywords with OR', () => {
const result = tokenizeQueryForFts(['hello', 'world'])
assert.ok(result.includes('OR'))
})
it('handles Chinese keywords', () => {
const result = tokenizeQueryForFts(['今天开心'])
assert.ok(result.length > 0)
})
})
@@ -3,7 +3,7 @@
*
* Bug history: when no custom dict existed on disk, every getJieba() call
* invalidated the cache and rebuilt a Jieba instance, making hot paths
* (FTS indexing tokenizes every message) rebuild jieba per call and slowing
* (bulk NLP analysis tokenizes many messages) rebuild jieba per call and slowing
* imports by seconds. The cache must only invalidate when the on-disk dict
* state actually changes (dict added or removed).
*/
@@ -1,75 +0,0 @@
/**
* FTS5 tokenizer — shared implementation.
*
* Unlike NLP word-frequency analysis, FTS tokenization keeps all tokens
* (no POS filtering, no stopwords) to maximize recall in search scenarios.
*
* Uses jieba for Chinese (naturally handles mixed CJK/Latin text),
* falls back to whitespace split when jieba is unavailable.
*/
import { getJieba } from './segmenter'
export function tokenizeForFts(text: string | null | undefined): string {
if (!text || text.trim().length === 0) return ''
try {
const jieba = getJieba()
const tokens = jieba.cut(text, false)
return tokens
.map((t) => t.trim().toLowerCase())
.filter((t) => t.length > 0)
.join(' ')
} catch {
return fallbackTokenize(text)
}
}
function fallbackTokenize(text: string): string {
return text
.toLowerCase()
.split(/\s+/)
.filter((t) => t.length > 0)
.join(' ')
}
function escapeToken(token: string): string {
return `"${token.replace(/"/g, '""')}"`
}
/**
* Convert a list of user search keywords into an FTS5 MATCH expression.
*
* - Single keyword: tokens joined with AND (all must appear)
* - Multiple keywords: groups joined with OR (any match)
*/
export function tokenizeQueryForFts(keywords: string[]): string {
if (keywords.length === 0) return ''
const groups = keywords
.map((kw) => {
const trimmed = kw.trim()
if (!trimmed) return ''
try {
const jieba = getJieba()
const tokens = jieba
.cut(trimmed, false)
.map((t) => t.trim().toLowerCase())
.filter((t) => t.length > 0)
if (tokens.length === 0) return ''
if (tokens.length === 1) return escapeToken(tokens[0])
return tokens.map(escapeToken).join(' ')
} catch {
const simple = trimmed.toLowerCase().trim()
return simple ? escapeToken(simple) : ''
}
})
.filter((g) => g.length > 0)
if (groups.length === 0) return ''
if (groups.length === 1) return groups[0]
return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ')
}
-3
View File
@@ -20,9 +20,6 @@ export {
// 词频统计
export { computeWordFrequency, segmentText } from './word-frequency'
// FTS tokenizer
export { tokenizeForFts, tokenizeQueryForFts } from './fts-tokenizer'
// 词库管理
export {
isDictDownloaded,
+1 -1
View File
@@ -69,7 +69,7 @@ export function getJieba(dictType: DictType = 'default'): JiebaInstance {
if (cached) {
// Only invalidate when the on-disk dict state changed since the instance
// was built (dict added or removed). Instances built without a disk dict
// must stay cached, otherwise hot paths (e.g. FTS indexing) would rebuild
// must stay cached, otherwise bulk NLP hot paths would rebuild
// a Jieba instance per call.
if (cached.fromDiskDict === existsDictOnDisk(effectiveType)) return cached.instance
jiebaInstances.delete(effectiveType)
+1 -1
View File
@@ -26,7 +26,7 @@ export interface ParsedMeta {
export interface ParseProgress {
/** 阶段 */
// 导入流程会复用解析进度结构,因此这里补充导入阶段枚举:
// saving = 消息批量写库,indexing = 建索引/FTS/会话分段等后处理。
// saving = 消息批量写库,indexing = 建普通索引/会话分段等后处理。
stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'
/** 已读取字节数 */
bytesRead: number
@@ -58,8 +58,7 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
export const deepSearchMessagesTool: ToolDefinition = {
name: 'deep_search_messages',
description:
'深度搜索消息(LIKE 子串匹配,速度较慢但不会遗漏),适用于全文搜索精确匹配或 FTS 搜索无结果时的补充搜索。',
description: '深度搜索消息(LIKE 子串匹配),适用于需要精确文本片段匹配的场景。',
inputSchema,
handler,
category: 'core',