refactor: unify message dedup key generator in @openchatlab/core

Extract canonical generateMessageKey (SHA-256 + base64url) into core.
Electron tempDb, merger, and server importer now share the same
algorithm, eliminating the server's divergent hex-truncated variant.

Co-authored-by: Cursor <cursoragent@cursor.com>

electron/main/merger/index.ts

@@ -5,7 +5,7 @@
 
 import * as fs from 'fs'
 import * as path from 'path'
-import { createHash } from 'crypto'
+import { generateMessageKey } from '@openchatlab/core'
 import { parseFileSync, detectFormat } from '../parser'
 import { importData } from '../database/core'
 import { TempDbReader } from './tempCache'
@@ -70,24 +70,8 @@ export async function parseFileInfo(filePath: string): Promise<FileParseInfo> {
   }
 }
 
-/**
- * 生成消息的唯一标识（用于去重和冲突检测）
- */
 function getMessageKey(msg: ParsedMessage, senderPlatformIdOverride?: string): string {
-  // 合并链路的去重语义需要和增量导入保持一致，否则两条链路会对重复消息得出不同结论。
-  const normalizedContent = msg.content || null
-  const senderPlatformId = senderPlatformIdOverride || msg.senderPlatformId
-  const hash = createHash('sha256')
-  hash.update(String(msg.timestamp))
-  hash.update('\0')
-  hash.update(senderPlatformId)
-  hash.update('\0')
-  hash.update(normalizedContent === null ? 'null' : 'text')
-  hash.update('\0')
-  if (normalizedContent !== null) {
-    hash.update(normalizedContent)
-  }
-  return hash.digest('base64url')
+  return generateMessageKey(msg.timestamp, senderPlatformIdOverride || msg.senderPlatformId, msg.content ?? null)
 }
 
 function getParsedMessageDisplayName(msg: ParsedMessage): string {

electron/main/worker/import/tempDb.ts

@@ -4,7 +4,6 @@
  */
 
 import Database from 'better-sqlite3'
-import { createHash } from 'crypto'
 import * as fs from 'fs'
 import * as path from 'path'
 
@@ -83,23 +82,5 @@ export function cleanupTempDatabase(dbPath: string): void {
   }
 }
 
-/**
- * 生成消息去重键
- * 使用固定长度哈希作为去重标识。
- * 直接用 content.length 会误判等长不同内容，直接用原文又会让 Set 长期持有大文本。
- */
-export function generateMessageKey(timestamp: number, senderPlatformId: string, content: string | null): string {
-  // 去重 key 需要和当前写库语义一致，空字符串在存储层会被折叠成 null。
-  const normalizedContent = content || null
-  const hash = createHash('sha256')
-  hash.update(String(timestamp))
-  hash.update('\0')
-  hash.update(senderPlatformId)
-  hash.update('\0')
-  hash.update(normalizedContent === null ? 'null' : 'text')
-  hash.update('\0')
-  if (normalizedContent !== null) {
-    hash.update(normalizedContent)
-  }
-  return hash.digest('base64url')
-}
+// Re-export the canonical dedup key generator from core
+export { generateMessageKey } from '@openchatlab/core'

packages/core/src/import/__tests__/dedup.test.ts

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { generateMessageKey } from '../dedup'
+
+describe('generateMessageKey', () => {
+  const ts = 1710000000
+  const sender = 'user-1'
+
+  it('empty string and null produce the same key (storage normalization)', () => {
+    const keyEmpty = generateMessageKey(ts, sender, '')
+    const keyNull = generateMessageKey(ts, sender, null)
+    assert.equal(keyEmpty, keyNull)
+  })
+
+  it('different content produces different keys even with the same length', () => {
+    const a = generateMessageKey(ts, sender, '你好啊')
+    const b = generateMessageKey(ts, sender, '再见呀')
+    assert.notEqual(a, b)
+  })
+
+  it('different timestamps produce different keys', () => {
+    const a = generateMessageKey(ts, sender, 'hello')
+    const b = generateMessageKey(ts + 1, sender, 'hello')
+    assert.notEqual(a, b)
+  })
+
+  it('different senders produce different keys', () => {
+    const a = generateMessageKey(ts, 'alice', 'hello')
+    const b = generateMessageKey(ts, 'bob', 'hello')
+    assert.notEqual(a, b)
+  })
+
+  it('returns a base64url string (no +, /, =)', () => {
+    const key = generateMessageKey(ts, sender, 'test content')
+    assert.ok(key.length > 0)
+    assert.ok(!/[+/=]/.test(key), `key should be base64url but got: ${key}`)
+  })
+
+  it('is deterministic for identical inputs', () => {
+    const a = generateMessageKey(ts, sender, 'same content')
+    const b = generateMessageKey(ts, sender, 'same content')
+    assert.equal(a, b)
+  })
+
+  it('null content and the literal string "null" produce different keys', () => {
+    const keyNull = generateMessageKey(ts, sender, null)
+    const keyLiteral = generateMessageKey(ts, sender, 'null')
+    assert.notEqual(keyNull, keyLiteral)
+  })
+})

packages/core/src/import/dedup.ts

@@ -0,0 +1,31 @@
+/**
+ * Canonical message deduplication key generator.
+ *
+ * Used by both Electron (worker import, merger) and Server (importer)
+ * to produce deterministic content-hash keys for messages that lack
+ * a platform_message_id.
+ *
+ * The key is a SHA-256 digest encoded as base64url (full length, ~43 chars).
+ * Segments are NUL-separated; content is preceded by a type discriminator
+ * so that null content and the literal string "null" hash differently.
+ *
+ * Empty-string content is normalized to null before hashing to match the
+ * storage-layer behavior where '' is folded to NULL in SQLite.
+ */
+
+import { createHash } from 'crypto'
+
+export function generateMessageKey(timestamp: number, senderPlatformId: string, content: string | null): string {
+  const normalizedContent = content || null
+  const hash = createHash('sha256')
+  hash.update(String(timestamp))
+  hash.update('\0')
+  hash.update(senderPlatformId)
+  hash.update('\0')
+  hash.update(normalizedContent === null ? 'null' : 'text')
+  hash.update('\0')
+  if (normalizedContent !== null) {
+    hash.update(normalizedContent)
+  }
+  return hash.digest('base64url')
+}

packages/core/src/import/index.ts

@@ -0,0 +1 @@
+export { generateMessageKey } from './dedup'

packages/core/src/index.ts

@@ -201,6 +201,9 @@ export {
   getBuiltinModelById,
 } from './ai'
 
+// Import utilities
+export { generateMessageKey } from './import'
+
 // Schema 与迁移
 export {
   CURRENT_SCHEMA_VERSION,

packages/server/src/import/importer.ts

@@ -8,7 +8,7 @@
 import * as fs from 'fs'
 import * as crypto from 'crypto'
 import type { DatabaseAdapter } from '@openchatlab/core'
-import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA } from '@openchatlab/core'
+import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey } from '@openchatlab/core'
 import type { DatabaseManager } from '@openchatlab/node-runtime'
 import { openBetterSqliteDatabase } from '@openchatlab/node-runtime'
 import type { ParsedData, ImportMessage } from './chatlab-reader'
@@ -35,11 +35,6 @@ function generateSessionId(): string {
   return `chat_${ts}_${rand}`
 }
 
-function generateMessageKey(ts: number, senderPlatformId: string, content: string | null): string {
-  const raw = `${ts}|${senderPlatformId}|${content ?? ''}`
-  return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 16)
-}
-
 /**
  * 新建导入：创建新的会话数据库并导入全部数据
  */