diff --git a/electron/main/merger/index.ts b/electron/main/merger/index.ts index 2bf35159..3a7c37cd 100644 --- a/electron/main/merger/index.ts +++ b/electron/main/merger/index.ts @@ -5,7 +5,7 @@ import * as fs from 'fs' import * as path from 'path' -import { createHash } from 'crypto' +import { generateMessageKey } from '@openchatlab/core' import { parseFileSync, detectFormat } from '../parser' import { importData } from '../database/core' import { TempDbReader } from './tempCache' @@ -70,24 +70,8 @@ export async function parseFileInfo(filePath: string): Promise { } } -/** - * 生成消息的唯一标识(用于去重和冲突检测) - */ function getMessageKey(msg: ParsedMessage, senderPlatformIdOverride?: string): string { - // 合并链路的去重语义需要和增量导入保持一致,否则两条链路会对重复消息得出不同结论。 - const normalizedContent = msg.content || null - const senderPlatformId = senderPlatformIdOverride || msg.senderPlatformId - const hash = createHash('sha256') - hash.update(String(msg.timestamp)) - hash.update('\0') - hash.update(senderPlatformId) - hash.update('\0') - hash.update(normalizedContent === null ? 'null' : 'text') - hash.update('\0') - if (normalizedContent !== null) { - hash.update(normalizedContent) - } - return hash.digest('base64url') + return generateMessageKey(msg.timestamp, senderPlatformIdOverride || msg.senderPlatformId, msg.content ?? null) } function getParsedMessageDisplayName(msg: ParsedMessage): string { diff --git a/electron/main/worker/import/tempDb.ts b/electron/main/worker/import/tempDb.ts index 03d01e15..3a9c80cb 100644 --- a/electron/main/worker/import/tempDb.ts +++ b/electron/main/worker/import/tempDb.ts @@ -4,7 +4,6 @@ */ import Database from 'better-sqlite3' -import { createHash } from 'crypto' import * as fs from 'fs' import * as path from 'path' @@ -83,23 +82,5 @@ export function cleanupTempDatabase(dbPath: string): void { } } -/** - * 生成消息去重键 - * 使用固定长度哈希作为去重标识。 - * 直接用 content.length 会误判等长不同内容,直接用原文又会让 Set 长期持有大文本。 - */ -export function generateMessageKey(timestamp: number, senderPlatformId: string, content: string | null): string { - // 去重 key 需要和当前写库语义一致,空字符串在存储层会被折叠成 null。 - const normalizedContent = content || null - const hash = createHash('sha256') - hash.update(String(timestamp)) - hash.update('\0') - hash.update(senderPlatformId) - hash.update('\0') - hash.update(normalizedContent === null ? 'null' : 'text') - hash.update('\0') - if (normalizedContent !== null) { - hash.update(normalizedContent) - } - return hash.digest('base64url') -} +// Re-export the canonical dedup key generator from core +export { generateMessageKey } from '@openchatlab/core' diff --git a/packages/core/src/import/__tests__/dedup.test.ts b/packages/core/src/import/__tests__/dedup.test.ts new file mode 100644 index 00000000..d1afaa61 --- /dev/null +++ b/packages/core/src/import/__tests__/dedup.test.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { generateMessageKey } from '../dedup' + +describe('generateMessageKey', () => { + const ts = 1710000000 + const sender = 'user-1' + + it('empty string and null produce the same key (storage normalization)', () => { + const keyEmpty = generateMessageKey(ts, sender, '') + const keyNull = generateMessageKey(ts, sender, null) + assert.equal(keyEmpty, keyNull) + }) + + it('different content produces different keys even with the same length', () => { + const a = generateMessageKey(ts, sender, '你好啊') + const b = generateMessageKey(ts, sender, '再见呀') + assert.notEqual(a, b) + }) + + it('different timestamps produce different keys', () => { + const a = generateMessageKey(ts, sender, 'hello') + const b = generateMessageKey(ts + 1, sender, 'hello') + assert.notEqual(a, b) + }) + + it('different senders produce different keys', () => { + const a = generateMessageKey(ts, 'alice', 'hello') + const b = generateMessageKey(ts, 'bob', 'hello') + assert.notEqual(a, b) + }) + + it('returns a base64url string (no +, /, =)', () => { + const key = generateMessageKey(ts, sender, 'test content') + assert.ok(key.length > 0) + assert.ok(!/[+/=]/.test(key), `key should be base64url but got: ${key}`) + }) + + it('is deterministic for identical inputs', () => { + const a = generateMessageKey(ts, sender, 'same content') + const b = generateMessageKey(ts, sender, 'same content') + assert.equal(a, b) + }) + + it('null content and the literal string "null" produce different keys', () => { + const keyNull = generateMessageKey(ts, sender, null) + const keyLiteral = generateMessageKey(ts, sender, 'null') + assert.notEqual(keyNull, keyLiteral) + }) +}) diff --git a/packages/core/src/import/dedup.ts b/packages/core/src/import/dedup.ts new file mode 100644 index 00000000..bc77aacc --- /dev/null +++ b/packages/core/src/import/dedup.ts @@ -0,0 +1,31 @@ +/** + * Canonical message deduplication key generator. + * + * Used by both Electron (worker import, merger) and Server (importer) + * to produce deterministic content-hash keys for messages that lack + * a platform_message_id. + * + * The key is a SHA-256 digest encoded as base64url (full length, ~43 chars). + * Segments are NUL-separated; content is preceded by a type discriminator + * so that null content and the literal string "null" hash differently. + * + * Empty-string content is normalized to null before hashing to match the + * storage-layer behavior where '' is folded to NULL in SQLite. + */ + +import { createHash } from 'crypto' + +export function generateMessageKey(timestamp: number, senderPlatformId: string, content: string | null): string { + const normalizedContent = content || null + const hash = createHash('sha256') + hash.update(String(timestamp)) + hash.update('\0') + hash.update(senderPlatformId) + hash.update('\0') + hash.update(normalizedContent === null ? 'null' : 'text') + hash.update('\0') + if (normalizedContent !== null) { + hash.update(normalizedContent) + } + return hash.digest('base64url') +} diff --git a/packages/core/src/import/index.ts b/packages/core/src/import/index.ts new file mode 100644 index 00000000..1bbb5695 --- /dev/null +++ b/packages/core/src/import/index.ts @@ -0,0 +1 @@ +export { generateMessageKey } from './dedup' diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index eeae3663..dc7549c9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -201,6 +201,9 @@ export { getBuiltinModelById, } from './ai' +// Import utilities +export { generateMessageKey } from './import' + // Schema 与迁移 export { CURRENT_SCHEMA_VERSION, diff --git a/packages/server/src/import/importer.ts b/packages/server/src/import/importer.ts index 45bd2fdc..43e72ec8 100644 --- a/packages/server/src/import/importer.ts +++ b/packages/server/src/import/importer.ts @@ -8,7 +8,7 @@ import * as fs from 'fs' import * as crypto from 'crypto' import type { DatabaseAdapter } from '@openchatlab/core' -import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA } from '@openchatlab/core' +import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey } from '@openchatlab/core' import type { DatabaseManager } from '@openchatlab/node-runtime' import { openBetterSqliteDatabase } from '@openchatlab/node-runtime' import type { ParsedData, ImportMessage } from './chatlab-reader' @@ -35,11 +35,6 @@ function generateSessionId(): string { return `chat_${ts}_${rand}` } -function generateMessageKey(ts: number, senderPlatformId: string, content: string | null): string { - const raw = `${ts}|${senderPlatformId}|${content ?? ''}` - return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 16) -} - /** * 新建导入:创建新的会话数据库并导入全部数据 */