feat(analysis): recognize Douyin export emoji codes

补充抖音导出表情代码识别表，让文本分析把 [微笑]、[星星眼]、[666]、[V5]、[kisskiss] 等带方括号的抖音表情代码当作表情，而不是普通文本词汇。

新增平台表情代码数据模块 bracket-emoji，集中管理微信经典表情映射与从抖音导出中收集到的表情代码名（含此前的 [躺平]、[强壮]）；cleanText 按 [代码]/【代码】 形式识别已知平台表情代码并在清洗时剥离。此前非中文名的抖音表情代码（如 [V5]、[ok]、[kisskiss]）会被当作普通词汇进入词频统计；语言偏好分析的分词清洗也改用与词频一致的 cleanText，表情代码不再混入高频词，而口头禅与短语统计仍保留 [表情] 原始形式。

补充脱敏合成数据回归测试，覆盖括号格式文本清洗、Node 与 Browser 词频统计以及语言偏好分析；新测试在识别逻辑缺失时会失败。

(cherry picked from commit c80885e0e3)

packages/core/src/nlp/bracket-emoji.ts

@@ -0,0 +1,290 @@
+/**
+ * Platform bracket emoji code tables (pure data, platform-agnostic).
+ *
+ * Chat exports render platform-specific emoji as bracketed code names such as
+ * `[微笑]`. Text analysis must tell these emoji codes apart from user-authored
+ * text, so each table below lists the code names observed in real exports of
+ * one platform. The WeChat table keeps its classic semantic mapping to Unicode
+ * emoji; other platforms only need name recognition for text cleaning.
+ */
+
+/**
+ * WeChat classic bracket emoji codes mapped to their Unicode equivalents.
+ * Kept as a mapping because the converted emoji may carry meaning beyond
+ * plain recognition.
+ */
+export const WECHAT_BRACKET_EMOJI_MAP: Record<string, string> = {
+  破涕为笑: '😂',
+  微笑: '🙂',
+  呲牙: '😁',
+  大笑: '😄',
+  笑哭: '😂',
+  流泪: '😢',
+  捂脸: '🤦',
+  发呆: '😳',
+  害羞: '😊',
+  调皮: '😜',
+  色: '😍',
+  惊讶: '😮',
+  撇嘴: '😒',
+  难过: '😞',
+  酷: '😎',
+  抓狂: '😫',
+  吐: '🤮',
+  偷笑: '🤭',
+  可爱: '😊',
+  白眼: '🙄',
+  傲慢: '😤',
+  饥饿: '😋',
+  困: '😴',
+  惊恐: '😱',
+  流汗: '😅',
+  憨笑: '😄',
+  悠闲: '😌',
+  奋斗: '💪',
+  咒骂: '😡',
+  疑问: '❓',
+  嘘: '🤫',
+  晕: '😵',
+  衰: '😞',
+  敲打: '👊',
+  再见: '👋',
+  擦汗: '😅',
+  抠鼻: '👃',
+  鼓掌: '👏',
+  坏笑: '😏',
+  左哼哼: '😤',
+  右哼哼: '😤',
+  哈欠: '🥱',
+  鄙视: '😒',
+  委屈: '🥺',
+  快哭了: '😢',
+  阴险: '😏',
+  亲亲: '😘',
+  吓: '😱',
+  可怜: '🥺',
+  菜刀: '🔪',
+  西瓜: '🍉',
+  啤酒: '🍺',
+  篮球: '🏀',
+  乒乓: '🏓',
+  咖啡: '☕',
+  饭: '🍚',
+  猪头: '🐷',
+  玫瑰: '🌹',
+  凋谢: '🥀',
+  爱心: '❤️',
+  心碎: '💔',
+  蛋糕: '🎂',
+  闪电: '⚡',
+  炸弹: '💣',
+  刀: '🔪',
+  足球: '⚽',
+  便便: '💩',
+  月亮: '🌙',
+  太阳: '☀️',
+  礼物: '🎁',
+  拥抱: '🤗',
+  强: '👍',
+  弱: '👎',
+  握手: '🤝',
+  胜利: '✌️',
+  抱拳: '🙏',
+  勾引: '☝️',
+  拳头: '✊',
+  差劲: '👎',
+  爱你: '🤟',
+  NO: '🙅',
+  OK: '👌',
+}
+
+/**
+ * Bracket emoji code names collected from Douyin chat exports, including the
+ * `[躺平]` and `[强壮]` codes covered by earlier system-message separation
+ * work.
+ */
+const DOUYIN_BRACKET_EMOJI_NAMES: readonly string[] = [
+  '微笑',
+  '色',
+  '发呆',
+  '撇嘴',
+  '流泪',
+  '捂脸',
+  '发怒',
+  '竖牙',
+  '哈哈',
+  '害羞',
+  '调皮',
+  '呲牙',
+  '亲亲',
+  '比心',
+  '鼓掌',
+  '惊讶',
+  '赞',
+  '憨笑',
+  '拥抱',
+  '抱抱',
+  '玫瑰',
+  '尴尬',
+  '流汗',
+  '握手',
+  '星星眼',
+  '杀马特',
+  '黄脸干杯',
+  '裸奔',
+  '举手',
+  '挥手',
+  '绿火花',
+  '笑脸',
+  '哭',
+  '吐舌',
+  '小狗',
+  '送花',
+  '爱心',
+  '手',
+  '贴贴',
+  '灵机一动',
+  '大笑',
+  '机智',
+  '送心',
+  '666',
+  '闭嘴',
+  '看我来',
+  '我一起加油',
+  '寒心',
+  '震惊',
+  '落',
+  '囧',
+  '疑问',
+  '问号',
+  '泣不成声',
+  '小鼓掌',
+  '大金牙',
+  '偷笑',
+  '右看',
+  '思考',
+  '吐血',
+  '可怜',
+  '得意',
+  '委屈',
+  '笑哭',
+  '抓狂',
+  '恐惧',
+  '快哭了',
+  '翻白眼',
+  '互粉',
+  '我想静静',
+  '鄙视',
+  '飞吻',
+  '再见',
+  '听歌',
+  '求抱抱',
+  '绝望的凝视',
+  '不失礼貌的微笑',
+  '不看',
+  '揉开',
+  '干饭人',
+  '庆祝',
+  '呆无辜',
+  '白眼',
+  '猪头',
+  '冷漠',
+  '暗中观察',
+  '二哈',
+  '黑脸',
+  '展开说说',
+  '蜜蜂狗',
+  '呆头',
+  '摸头',
+  '腼腆',
+  '做鬼脸',
+  '强',
+  '如花',
+  '吓',
+  '惊喜',
+  '打',
+  '奋斗',
+  '吐彩虹',
+  '大英',
+  '吓唬',
+  '惊恐',
+  '困',
+  '斜眼',
+  '阴险',
+  '咒骂',
+  '吃瓜',
+  '群众',
+  '绿帽子',
+  '求求了',
+  '眼含热泪',
+  '宕',
+  '气',
+  '好开心',
+  '不是吧',
+  '驾',
+  '躺平',
+  '不你不想',
+  '一头乱麻',
+  'kisskiss',
+  '你不大行',
+  '嗅',
+  '苦笑',
+  '苦涩',
+  '坚强',
+  '落泪',
+  '求机位',
+  '黄脸',
+  '点赞',
+  '精选',
+  '硬拳',
+  'OOK',
+  '左上',
+  '抱拳',
+  '勾引',
+  '拳头',
+  '附',
+  '胜利',
+  '右边',
+  '右',
+  '躲',
+  '垃圾',
+  '啤酒',
+  '蛋糕',
+  '礼物',
+  '撒花',
+  '加一',
+  '呸',
+  'ok',
+  'V5',
+  '发',
+  '18',
+  '禁',
+  '西瓜',
+  '加鸡腿',
+  '太阳',
+  '月亮',
+  '给跪了',
+  '打心',
+  '打call',
+  '栓Q',
+  '雪花',
+  '气球',
+  '福',
+  'candy',
+  '糖葫芦',
+  '鞭炮',
+  '元宝',
+  '灯笼',
+  '巧克力',
+  '威哥',
+  '棒棒糖',
+  '纸飞机',
+  '粽子',
+  '强壮',
+]
+
+/**
+ * Every known platform bracket emoji name. Text cleaning recognizes these
+ * names as emoji placeholders instead of user-authored words.
+ */
+export const KNOWN_BRACKET_EMOJI_NAMES: ReadonlySet<string> = new Set([...DOUYIN_BRACKET_EMOJI_NAMES])

packages/core/src/nlp/text-utils.test.ts

@@ -18,6 +18,10 @@ describe('cleanText', () => {
     ['removes bracketed chat emoji placeholders before tokenization', '今天[破涕为笑][微笑][呲牙]很好', '今天 很好'],
     ['removes unknown short bracketed emoji placeholders', '收到[旺柴]马上来', '收到 马上来'],
     ['removes mapped emoji placeholders with variation selectors', '送你[爱心][太阳]', '送你'],
+    ['removes exact bracketed Douyin emoji codes', '太累了[躺平]，一起[强壮]', '太累了 一起'],
+    ['removes full-width bracketed emoji codes', '【微笑】收到', '收到'],
+    ['removes non-CJK Douyin emoji codes as emoji', '真的[V5]呀[666]', '真的 呀'],
+    ['removes collected Douyin emoji codes in mixed text', '发一个[kisskiss]给你', '发一个 给你'],
     ['keeps ordinary non-bracketed words', '破涕为笑 微笑 呲牙', '破涕为笑 微笑 呲牙'],
     ['keeps non-CJK bracketed words as regular text', 'please check [report]', 'please check report'],
   ] as const

packages/core/src/nlp/text-utils.ts

@@ -2,6 +2,8 @@
  * 文本处理工具（纯函数，平台无关）
  */
 
+import { KNOWN_BRACKET_EMOJI_NAMES, WECHAT_BRACKET_EMOJI_MAP } from './bracket-emoji'
+
 const EMOJI_REGEX = /[\u{1F300}-\u{1FAFF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu
 const EMOJI_VARIATION_SELECTOR_REGEX = /\u{FE0F}/gu
 const PUNCTUATION_REGEX = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~，。！？、；：""''（）【】《》…—～·\s]/g
@@ -29,91 +31,6 @@ const BRACKET_EMOJI_PLACEHOLDER_REGEX =
   /(?:\[|【)([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Letter}\p{Number}_-]{1,16})(?:\]|】)/gu
 const CJK_TEXT_REGEX = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u
 
-const BRACKET_EMOJI_MAP: Record<string, string> = {
-  破涕为笑: '😂',
-  微笑: '🙂',
-  呲牙: '😁',
-  大笑: '😄',
-  笑哭: '😂',
-  流泪: '😢',
-  捂脸: '🤦',
-  发呆: '😳',
-  害羞: '😊',
-  调皮: '😜',
-  色: '😍',
-  惊讶: '😮',
-  撇嘴: '😒',
-  难过: '😞',
-  酷: '😎',
-  抓狂: '😫',
-  吐: '🤮',
-  偷笑: '🤭',
-  可爱: '😊',
-  白眼: '🙄',
-  傲慢: '😤',
-  饥饿: '😋',
-  困: '😴',
-  惊恐: '😱',
-  流汗: '😅',
-  憨笑: '😄',
-  悠闲: '😌',
-  奋斗: '💪',
-  咒骂: '😡',
-  疑问: '❓',
-  嘘: '🤫',
-  晕: '😵',
-  衰: '😞',
-  敲打: '👊',
-  再见: '👋',
-  擦汗: '😅',
-  抠鼻: '👃',
-  鼓掌: '👏',
-  坏笑: '😏',
-  左哼哼: '😤',
-  右哼哼: '😤',
-  哈欠: '🥱',
-  鄙视: '😒',
-  委屈: '🥺',
-  快哭了: '😢',
-  阴险: '😏',
-  亲亲: '😘',
-  吓: '😱',
-  可怜: '🥺',
-  菜刀: '🔪',
-  西瓜: '🍉',
-  啤酒: '🍺',
-  篮球: '🏀',
-  乒乓: '🏓',
-  咖啡: '☕',
-  饭: '🍚',
-  猪头: '🐷',
-  玫瑰: '🌹',
-  凋谢: '🥀',
-  爱心: '❤️',
-  心碎: '💔',
-  蛋糕: '🎂',
-  闪电: '⚡',
-  炸弹: '💣',
-  刀: '🔪',
-  足球: '⚽',
-  便便: '💩',
-  月亮: '🌙',
-  太阳: '☀️',
-  礼物: '🎁',
-  拥抱: '🤗',
-  强: '👍',
-  弱: '👎',
-  握手: '🤝',
-  胜利: '✌️',
-  抱拳: '🙏',
-  勾引: '☝️',
-  拳头: '✊',
-  差劲: '👎',
-  爱你: '🤟',
-  NO: '🙅',
-  OK: '👌',
-}
-
 /**
  * 清理文本：移除 URL、@提及、表情、标点等
  */
@@ -123,7 +40,8 @@ export function cleanText(text: string): string {
     .replace(MENTION_REGEX, ' ')
     .replace(SYSTEM_PLACEHOLDER_REGEX, ' ')
     .replace(BRACKET_EMOJI_PLACEHOLDER_REGEX, (match, name: string) => {
-      if (BRACKET_EMOJI_MAP[name]) return BRACKET_EMOJI_MAP[name]
+      if (WECHAT_BRACKET_EMOJI_MAP[name]) return WECHAT_BRACKET_EMOJI_MAP[name]
+      if (KNOWN_BRACKET_EMOJI_NAMES.has(name)) return ' '
       return CJK_TEXT_REGEX.test(name) ? ' ' : match
     })
     .replace(EMOJI_REGEX, ' ')

packages/core/src/query/__tests__/browser-word-frequency.test.ts

@@ -161,6 +161,26 @@ describe('getBrowserWordFrequency', () => {
     assert.equal(result.totalMessages, 4)
   })
 
+  it('treats Douyin emoji codes as emoji instead of words', () => {
+    const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
+    insert.run(13, 1, 1300, 0, '[kisskiss]')
+    insert.run(14, 1, 1400, 5, '[V5]')
+
+    const result = getBrowserWordFrequency(database, {
+      sessionId: 'session-one',
+      locale: 'en-US',
+      topN: 20,
+      minCount: 1,
+      posFilterMode: 'all',
+      enableStopwords: false,
+    })
+
+    const words = result.words.map((word) => word.word)
+    assert.ok(!words.includes('kisskiss'))
+    assert.ok(!words.includes('v5'))
+    assert.equal(result.totalMessages, 5)
+  })
+
   it('applies member, time, excluded-word, and excluded-message filters', () => {
     const result = getBrowserWordFrequency(database, {
       sessionId: 'session-one',

packages/core/src/query/advanced/languagePreference.ts

@@ -7,7 +7,7 @@
 
 import { MessageType, type TimeFilter } from '@openchatlab/shared-types'
 import type { DatabaseAdapter } from '../../interfaces'
-import { isSystemMessageContent, stripMediaPlaceholders, stripVoiceTranscriptionPrefix } from '../../nlp'
+import { cleanText, isSystemMessageContent, stripVoiceTranscriptionPrefix } from '../../nlp'
 import { buildTimeFilter } from '../filters'
 import { isSystemPlaceholderContent } from './text-filters'
 
@@ -39,24 +39,9 @@ const ADJ_TAGS = new Set(['a', 'an', 'ad', 'ag'])
 const ADV_TAGS = new Set(['d'])
 const MODAL_TAGS = new Set(['y', 'e'])
 
-const RE_URL = /https?:\/\/[^\s]+/g
-const RE_MENTION = /@[^\s@]+/g
-const RE_EMOJI =
-  /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu
-const RE_PUNCTUATION = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~，。！？、；：""''（）【】《》…—～·\s]/g
 const RE_PURE_NUMBER = /^\d+$/
 const LANGUAGE_ANALYSIS_MESSAGE_TYPES = [MessageType.TEXT, MessageType.EMOJI].join(', ')
 
-function cleanTextForNlp(text: string): string {
-  return stripMediaPlaceholders(text)
-    .replace(RE_URL, ' ')
-    .replace(RE_MENTION, ' ')
-    .replace(RE_EMOJI, ' ')
-    .replace(RE_PUNCTUATION, ' ')
-    .replace(/\s+/g, ' ')
-    .trim()
-}
-
 function countMatches(text: string, regex: RegExp): number {
   regex.lastIndex = 0
   const m = text.match(regex)
@@ -144,7 +129,7 @@ export function getLanguagePreferenceAnalysis(db: DatabaseAdapter, params: Langu
 
       if (trimmed.length >= 2) phraseFreq.set(trimmed, (phraseFreq.get(trimmed) || 0) + 1)
 
-      const cleaned = cleanTextForNlp(content)
+      const cleaned = cleanText(content)
       if (!cleaned) continue
 
       if (isChinese && nlpProvider) {

packages/core/src/query/advanced/repeat.test.ts

@@ -298,4 +298,22 @@ describe('getLanguagePreferenceAnalysis', () => {
       { content: '[躺平]', count: 2 },
     ])
   })
+
+  it('keeps emoji codes as phrases but excludes them from vocabulary', () => {
+    const db = createRowsDb([
+      { memberId: 1, name: 'Alice', content: '[微笑]' },
+      { memberId: 1, name: 'Alice', content: '[微笑]' },
+      { memberId: 1, name: 'Alice', content: '[V5]' },
+      { memberId: 1, name: 'Alice', content: '[V5]' },
+    ])
+
+    const result = getLanguagePreferenceAnalysis(db, { locale: 'zh-CN' })
+    const alice = result.members[0]
+
+    assert.deepEqual(alice?.catchphrases, [
+      { content: '[微笑]', count: 2 },
+      { content: '[V5]', count: 2 },
+    ])
+    assert.ok((alice?.topWords ?? []).every((word: { word: string }) => !['微笑', 'v5'].includes(word.word)))
+  })
 })

packages/node-runtime/src/nlp/word-frequency.test.ts

@@ -134,4 +134,24 @@ describe('computeWordFrequency excludeKeywords pushdown', () => {
     assert.ok(!words.includes('share_noise'))
     assert.equal(result.totalMessages, 4)
   })
+
+  it('treats Douyin emoji codes as emoji instead of vocabulary', () => {
+    const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
+    insert.run(4, 1, 4000, 0, '[kisskiss]')
+    insert.run(5, 1, 5000, 5, '[V5]')
+
+    const result = computeWordFrequency(db, {
+      sessionId: 's1',
+      locale: 'en-US',
+      minCount: 1,
+      posFilterMode: 'all',
+      enableStopwords: false,
+    })
+
+    const words = result.words.map((word) => word.word)
+    assert.ok(!words.includes('kisskiss'))
+    assert.ok(!words.includes('v5'))
+    // Emoji messages stay eligible for analysis; they only contribute no words.
+    assert.equal(result.totalMessages, 5)
+  })
 })