fix(analysis): separate system messages from emoji phrases

区分抖音导出中的显式系统通知、分享占位和用户表情，避免系统提示进入口头禅及语言、词频统计。

新增统一的系统/分享前缀识别，覆盖 [系统]、[系统消息]、[分享内容] 及对应全角和英文形式；不误判 [强壮]、[躺平] 等表情标签。口头禅、语言偏好以及 Node/Browser 词频分析支持文本与表情消息，并在统计阶段排除系统通知和分享占位；原始聊天记录保持不变。

补充脱敏合成数据回归测试，覆盖系统通知、分享内容占位排除以及表情保留。

packages/core/src/index.ts

@@ -298,6 +298,7 @@ export {
   getStopwords,
   isStopword,
   cleanText,
+  isSystemMessageContent,
   isValidWord,
 } from './nlp'
 export type {

packages/core/src/nlp/index.ts

@@ -27,6 +27,7 @@ export { CHINESE_STOPWORDS, ENGLISH_STOPWORDS, JAPANESE_STOPWORDS, getStopwords,
 export {
   cleanText,
   isMediaPlaceholderContent,
+  isSystemMessageContent,
   isValidWord,
   stripMediaPlaceholders,
   stripVoiceTranscriptionPrefix,

packages/core/src/nlp/text-utils.test.ts

@@ -1,6 +1,11 @@
 import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
-import { cleanText, isMediaPlaceholderContent, stripVoiceTranscriptionPrefix } from './text-utils'
+import {
+  cleanText,
+  isMediaPlaceholderContent,
+  isSystemMessageContent,
+  stripVoiceTranscriptionPrefix,
+} from './text-utils'
 
 describe('cleanText', () => {
   const cases = [
@@ -32,4 +37,22 @@ describe('cleanText', () => {
     assert.equal(isMediaPlaceholderContent('[语音 3秒]'), true)
     assert.equal(isMediaPlaceholderContent('[语音 3秒] 有转写内容'), false)
   })
+
+  it('recognizes explicit system markers without treating bracketed emoji labels as system text', () => {
+    const cases = [
+      ['[系统] 对方赞了你分享的 图文', true],
+      ['【系统消息】你赞了对方分享的视频', true],
+      ['[System Message] shared a post', true],
+      ['[分享内容]', true],
+      ['【分享内容】', true],
+      ['[强壮]', false],
+      ['[躺平]', false],
+      ['[分享]', false],
+      ['我说“[系统]”这个词', false],
+    ] as const
+
+    for (const [input, expected] of cases) {
+      assert.equal(isSystemMessageContent(input), expected, input)
+    }
+  })
 })

packages/core/src/nlp/text-utils.ts

@@ -23,6 +23,8 @@ const VOICE_TRANSCRIPTION_PREFIX_REGEX = new RegExp(
   `^\\s*\\[(?:语音|Voice|Audio)${MEDIA_PLACEHOLDER_DURATION_PATTERN}\\]\\s*`,
   'i'
 )
+const SYSTEM_MESSAGE_PREFIX_REGEX =
+  /^\s*(?:\[(?:系统(?:消息)?|分享内容|system(?:\s+message)?)\]|【(?:系统(?:消息)?|分享内容|system(?:\s+message)?)】)\s*/iu
 const BRACKET_EMOJI_PLACEHOLDER_REGEX =
   /(?:\[|【)([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Letter}\p{Number}_-]{1,16})(?:\]|】)/gu
 const CJK_TEXT_REGEX = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u
@@ -141,6 +143,11 @@ export function stripVoiceTranscriptionPrefix(text: string): string {
   return text.replace(VOICE_TRANSCRIPTION_PREFIX_REGEX, '').trim()
 }
 
+/** Identify an explicit system/share marker without classifying bracketed emoji labels as system text. */
+export function isSystemMessageContent(text: string): boolean {
+  return SYSTEM_MESSAGE_PREFIX_REGEX.test(text)
+}
+
 /** Whether the complete value is a media placeholder rather than message text. */
 export function isMediaPlaceholderContent(text: string): boolean {
   return MEDIA_PLACEHOLDER_ONLY_REGEX.test(text.trim())

packages/core/src/query/__tests__/browser-word-frequency.test.ts

@@ -138,6 +138,29 @@ describe('getBrowserWordFrequency', () => {
     assert.ok(!words.includes('voice'))
   })
 
+  it('excludes explicit system notifications from word frequency', () => {
+    const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
+    insert.run(8, 1, 800, 0, '[系统] 对方赞了你分享的 systemnoise')
+    insert.run(9, 1, 900, 0, '[系统] 对方赞了你分享的 systemnoise')
+    insert.run(10, 1, 1000, 5, '[强壮]')
+    insert.run(11, 1, 1100, 0, '[分享内容] share_noise')
+    insert.run(12, 1, 1200, 0, '[分享内容] share_noise')
+
+    const result = getBrowserWordFrequency(database, {
+      sessionId: 'session-one',
+      locale: 'en-US',
+      topN: 20,
+      minCount: 1,
+      posFilterMode: 'all',
+      enableStopwords: false,
+    })
+
+    const words = result.words.map((word) => word.word)
+    assert.ok(!words.includes('systemnoise'))
+    assert.ok(!words.includes('share_noise'))
+    assert.equal(result.totalMessages, 4)
+  })
+
   it('applies member, time, excluded-word, and excluded-message filters', () => {
     const result = getBrowserWordFrequency(database, {
       sessionId: 'session-one',

packages/core/src/query/advanced/languagePreference.ts

@@ -5,9 +5,9 @@
  * 调用方负责提供对应平台的实现（Electron / Server 用 jieba，浏览器可降级）。
  */
 
-import type { TimeFilter } from '@openchatlab/shared-types'
+import { MessageType, type TimeFilter } from '@openchatlab/shared-types'
 import type { DatabaseAdapter } from '../../interfaces'
-import { stripMediaPlaceholders, stripVoiceTranscriptionPrefix } from '../../nlp'
+import { isSystemMessageContent, stripMediaPlaceholders, stripVoiceTranscriptionPrefix } from '../../nlp'
 import { buildTimeFilter } from '../filters'
 import { isSystemPlaceholderContent } from './text-filters'
 
@@ -45,6 +45,7 @@ const RE_EMOJI =
   /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu
 const RE_PUNCTUATION = /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~，。！？、；：""''（）【】《》…—～·\s]/g
 const RE_PURE_NUMBER = /^\d+$/
+const LANGUAGE_ANALYSIS_MESSAGE_TYPES = [MessageType.TEXT, MessageType.EMOJI].join(', ')
 
 function cleanTextForNlp(text: string): string {
   return stripMediaPlaceholders(text)
@@ -88,8 +89,7 @@ export function getLanguagePreferenceAnalysis(db: DatabaseAdapter, params: Langu
 
   const { clause, params: filterParams } = buildTimeFilter(timeFilter)
   let whereClause = clause
-  const textFilter =
-    " COALESCE(m.account_name, '') != '系统消息' AND msg.type = 0 AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2"
+  const textFilter = ` COALESCE(m.account_name, '') != '系统消息' AND msg.type IN (${LANGUAGE_ANALYSIS_MESSAGE_TYPES}) AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2`
   if (whereClause.includes('WHERE')) {
     whereClause += ' AND ' + textFilter
   } else {
@@ -108,7 +108,7 @@ export function getLanguagePreferenceAnalysis(db: DatabaseAdapter, params: Langu
   const memberMessages = new Map<number, { name: string; messages: string[] }>()
   for (const row of rows) {
     const content = stripVoiceTranscriptionPrefix(row.content)
-    if (!content || isSystemPlaceholderContent(content)) continue
+    if (!content || isSystemMessageContent(content) || isSystemPlaceholderContent(content)) continue
 
     let entry = memberMessages.get(row.memberId)
     if (!entry) {

packages/core/src/query/advanced/repeat.test.ts

@@ -1,6 +1,7 @@
 import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
 import Database from 'better-sqlite3'
+import { MessageType } from '@openchatlab/shared-types'
 import type { DatabaseAdapter, PreparedStatement, RunResult } from '../../interfaces'
 import { getLanguagePreferenceAnalysis } from './languagePreference'
 import { getCatchphraseAnalysis } from './repeat'
@@ -91,6 +92,7 @@ interface CatchphraseFixture {
   memberId: number
   content: string
   count: number
+  type?: number
 }
 
 function createCatchphraseDb(fixtures: CatchphraseFixture[]): DatabaseAdapter {
@@ -114,11 +116,17 @@ function createCatchphraseDb(fixtures: CatchphraseFixture[]): DatabaseAdapter {
       (2, 'bob', 'Bob', NULL);
   `)
 
-  const insert = database.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, 0, ?)')
+  const insert = database.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
   let messageId = 1
   for (const fixture of fixtures) {
     for (let index = 0; index < fixture.count; index += 1) {
-      insert.run(messageId, fixture.memberId, 1_700_000_000 + messageId, fixture.content)
+      insert.run(
+        messageId,
+        fixture.memberId,
+        1_700_000_000 + messageId,
+        fixture.type ?? MessageType.TEXT,
+        fixture.content
+      )
       messageId += 1
     }
   }
@@ -194,6 +202,24 @@ describe('getCatchphraseAnalysis', () => {
     assert.deepEqual(result.members[0]?.catchphrases, [{ content: '同一句测试转写', count: 2 }])
     db.close()
   })
+
+  it('filters explicit system notifications but keeps emoji labels', () => {
+    const db = createCatchphraseDb([
+      { memberId: 1, content: '[系统] 对方赞了你分享的 图文', count: 4 },
+      { memberId: 1, content: '[系统] 你赞了对方分享的 视频', count: 3 },
+      { memberId: 1, content: '[分享内容]', count: 5 },
+      { memberId: 1, content: '[强壮]', count: 4, type: MessageType.EMOJI },
+      { memberId: 1, content: '[躺平]', count: 3, type: MessageType.TEXT },
+    ])
+
+    const result = getCatchphraseAnalysis(db)
+
+    assert.deepEqual(result.members[0]?.catchphrases, [
+      { content: '[强壮]', count: 4 },
+      { content: '[躺平]', count: 3 },
+    ])
+    db.close()
+  })
 })
 
 describe('getLanguagePreferenceAnalysis', () => {
@@ -236,4 +262,26 @@ describe('getLanguagePreferenceAnalysis', () => {
       true
     )
   })
+
+  it('filters system notifications while retaining bracketed emoji phrases', () => {
+    const db = createRowsDb([
+      { memberId: 1, name: 'Alice', content: '[系统] 对方赞了你分享的 图文' },
+      { memberId: 1, name: 'Alice', content: '[系统] 对方赞了你分享的 图文' },
+      { memberId: 1, name: 'Alice', content: '[分享内容] share_noise' },
+      { memberId: 1, name: 'Alice', content: '[分享内容] share_noise' },
+      { memberId: 1, name: 'Alice', content: '[强壮]' },
+      { memberId: 1, name: 'Alice', content: '[强壮]' },
+      { memberId: 1, name: 'Alice', content: '[躺平]' },
+      { memberId: 1, name: 'Alice', content: '[躺平]' },
+    ])
+
+    const result = getLanguagePreferenceAnalysis(db, { locale: 'zh-CN' })
+    const alice = result.members.find((member: { name: string }) => member.name === 'Alice')
+
+    assert.equal(alice?.totalMessages, 4)
+    assert.deepEqual(alice?.catchphrases, [
+      { content: '[强壮]', count: 2 },
+      { content: '[躺平]', count: 2 },
+    ])
+  })
 })

packages/core/src/query/advanced/repeat.ts

@@ -2,9 +2,9 @@
  * 口头禅分析模块（平台无关）
  */
 
-import type { TimeFilter } from '@openchatlab/shared-types'
+import { MessageType, type TimeFilter } from '@openchatlab/shared-types'
 import type { DatabaseAdapter } from '../../interfaces'
-import { stripVoiceTranscriptionPrefix } from '../../nlp'
+import { isSystemMessageContent, stripVoiceTranscriptionPrefix } from '../../nlp'
 import { buildTimeFilter } from '../filters'
 import { isSystemPlaceholderContent } from './text-filters'
 
@@ -18,6 +18,7 @@ const VOICE_TRANSCRIPTION_SQL = `
         TRIM(msg.content) LIKE '[Audio] %' OR
         TRIM(msg.content) LIKE '[Audio %] %'
       `
+const CATCHPHRASE_MESSAGE_TYPES = [MessageType.TEXT, MessageType.EMOJI].join(', ')
 
 export interface CatchphraseItem {
   content: string
@@ -40,11 +41,9 @@ export function getCatchphraseAnalysis(db: DatabaseAdapter, filter?: TimeFilter)
 
   let whereClause = clause
   if (whereClause.includes('WHERE')) {
-    whereClause +=
-      " AND COALESCE(m.account_name, '') != '系统消息' AND msg.type = 0 AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2"
+    whereClause += ` AND COALESCE(m.account_name, '') != '系统消息' AND msg.type IN (${CATCHPHRASE_MESSAGE_TYPES}) AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2`
   } else {
-    whereClause =
-      " WHERE COALESCE(m.account_name, '') != '系统消息' AND msg.type = 0 AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2"
+    whereClause = ` WHERE COALESCE(m.account_name, '') != '系统消息' AND msg.type IN (${CATCHPHRASE_MESSAGE_TYPES}) AND msg.content IS NOT NULL AND LENGTH(TRIM(msg.content)) >= 2`
   }
 
   const rows = db
@@ -77,7 +76,7 @@ export function getCatchphraseAnalysis(db: DatabaseAdapter, filter?: TimeFilter)
 
   for (const row of rows) {
     const content = stripVoiceTranscriptionPrefix(row.content)
-    if (!content || isSystemPlaceholderContent(content)) continue
+    if (!content || isSystemMessageContent(content) || isSystemPlaceholderContent(content)) continue
 
     if (!memberMap.has(row.memberId)) {
       memberMap.set(row.memberId, {

packages/core/src/query/browser-word-frequency.ts

@@ -1,5 +1,13 @@
 import type { DatabaseAdapter } from '../interfaces'
-import { cleanText, isStopword, isValidWord, type WordFrequencyParams, type WordFrequencyResult } from '../nlp'
+import { MessageType } from '@openchatlab/shared-types'
+import {
+  cleanText,
+  isStopword,
+  isSystemMessageContent,
+  isValidWord,
+  type WordFrequencyParams,
+  type WordFrequencyResult,
+} from '../nlp'
 import { buildExcludeKeywordsConditions } from './message-sql'
 
 function tokenize(text: string, locale: string): string[] {
@@ -34,7 +42,7 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
   } = params
   const conditions = [
     "COALESCE(m.account_name, '') != '系统消息'",
-    'msg.type = 0',
+    `msg.type IN (${MessageType.TEXT}, ${MessageType.EMOJI})`,
     'msg.content IS NOT NULL',
     "TRIM(msg.content) != ''",
   ]
@@ -66,10 +74,11 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
        WHERE ${conditions.join(' AND ')}`
     )
     .all(...queryParams) as Array<{ content: string }>
+  const analyzableMessages = messages.filter((message) => !isSystemMessageContent(message.content))
   const excludedWords = new Set(excludeWords.map((word) => word.trim().toLocaleLowerCase(locale)).filter(Boolean))
   const frequencies = new Map<string, number>()
 
-  for (const message of messages) {
+  for (const message of analyzableMessages) {
     for (const rawWord of tokenize(message.content, locale)) {
       const word = rawWord.toLocaleLowerCase(locale)
       if (excludedWords.has(word)) continue
@@ -92,7 +101,7 @@ export function getBrowserWordFrequency(db: DatabaseAdapter, params: WordFrequen
       percentage: rankedTotal > 0 ? Math.round((count / rankedTotal) * 10_000) / 100 : 0,
     })),
     totalWords,
-    totalMessages: messages.length,
+    totalMessages: analyzableMessages.length,
     uniqueWords: filteredFrequencies.length,
   }
 }

packages/node-runtime/src/nlp/word-frequency.test.ts

@@ -112,4 +112,26 @@ describe('computeWordFrequency excludeKeywords pushdown', () => {
     })
     assert.equal(wildcard.totalMessages, 3)
   })
+
+  it('excludes explicit system notifications while keeping emoji messages eligible', () => {
+    const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
+    insert.run(4, 1, 4000, 0, '[系统] 你赞了对方分享的 systemnoise')
+    insert.run(5, 1, 5000, 0, '[系统] 你赞了对方分享的 systemnoise')
+    insert.run(6, 1, 6000, 5, '[强壮]')
+    insert.run(7, 1, 7000, 0, '[分享内容] share_noise')
+    insert.run(8, 1, 8000, 0, '[分享内容] share_noise')
+
+    const result = computeWordFrequency(db, {
+      sessionId: 's1',
+      locale: 'en-US',
+      minCount: 1,
+      posFilterMode: 'all',
+      enableStopwords: false,
+    })
+
+    const words = result.words.map((word) => word.word)
+    assert.ok(!words.includes('systemnoise'))
+    assert.ok(!words.includes('share_noise'))
+    assert.equal(result.totalMessages, 4)
+  })
 })

packages/node-runtime/src/nlp/word-frequency.ts

@@ -5,7 +5,8 @@
  * 供 Electron worker 和 Server HTTP 路由共用。
  */
 
-import { buildExcludeKeywordsConditions, type DatabaseAdapter } from '@openchatlab/core'
+import { buildExcludeKeywordsConditions, isSystemMessageContent, type DatabaseAdapter } from '@openchatlab/core'
+import { MessageType } from '@openchatlab/shared-types'
 import type {
   SupportedLocale,
   DictType,
@@ -44,7 +45,7 @@ function buildMessageQuery(
   }
 
   conditions.push("COALESCE(m.account_name, '') != '系统消息'")
-  conditions.push('msg.type = 0')
+  conditions.push(`msg.type IN (${MessageType.TEXT}, ${MessageType.EMOJI})`)
   conditions.push('msg.content IS NOT NULL')
   conditions.push("TRIM(msg.content) != ''")
 
@@ -83,7 +84,12 @@ export function computeWordFrequency(db: DatabaseAdapter, params: WordFrequencyP
     return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 }
   }
 
-  const texts = messages.map((m) => m.content)
+  const analyzableMessages = messages.filter((message) => !isSystemMessageContent(message.content))
+  if (analyzableMessages.length === 0) {
+    return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 }
+  }
+
+  const texts = analyzableMessages.map((m) => m.content)
 
   const segmentOptions = {
     minLength: minWordLength,
@@ -124,7 +130,7 @@ export function computeWordFrequency(db: DatabaseAdapter, params: WordFrequencyP
   return {
     words,
     totalWords: result.totalWords,
-    totalMessages: messages.length,
+    totalMessages: analyzableMessages.length,
     uniqueWords: result.uniqueWords,
     posTagStats,
   }