mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
refactor: extract preprocessor pipeline to @openchatlab/node-runtime
Move preprocessing (pipeline, types, builtin-rules, format utilities) from Electron to shared package. Electron side becomes thin wrappers.
This commit is contained in:
digua committed
1 parent
b195750f34
commit
76b4be1264
11 files changed
+800
-581
No files matched your search
@@ -19,3 +19,26 @@ export type {
|
||||
|
||||
// Tokenizer
|
||||
export { countTokens, countMessagesTokens } from './tokenizer'
|
||||
|
||||
// Preprocessor
|
||||
export type {
|
||||
PreprocessConfig,
|
||||
PreprocessableMessage,
|
||||
DesensitizeRule,
|
||||
TruncationStrategy,
|
||||
PreprocessLogger,
|
||||
} from './preprocessor'
|
||||
export {
|
||||
preprocessMessages,
|
||||
BUILTIN_DESENSITIZE_RULES,
|
||||
getDefaultRulesForLocale,
|
||||
mergeRulesForLocale,
|
||||
formatMessageCompact,
|
||||
formatTimeRange,
|
||||
formatToolResultAsText,
|
||||
anonymizeMessageNames,
|
||||
truncateFormattedMessages,
|
||||
isChineseLocale,
|
||||
i18nTexts,
|
||||
t,
|
||||
} from './preprocessor'
|
||||
@@ -0,0 +1,195 @@
|
||||
/**
|
||||
* 内置脱敏规则库
|
||||
* 按 locale 分组，支持通用规则和地区特定规则
|
||||
*/
|
||||
import type { DesensitizeRule } from './types'
|
||||
|
||||
export const BUILTIN_DESENSITIZE_RULES: DesensitizeRule[] = [
|
||||
// ==================== 中国 (zh-CN) ====================
|
||||
{
|
||||
id: 'cn_phone',
|
||||
label: 'desensitize.rules.cn_phone',
|
||||
pattern: '(?<!\\d)1[3-9]\\d{9}(?!\\d)',
|
||||
replacement: '[手机号]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['zh-CN'],
|
||||
},
|
||||
{
|
||||
id: 'cn_id_card',
|
||||
label: 'desensitize.rules.cn_id_card',
|
||||
pattern: '(?<!\\d)\\d{17}[\\dXx](?!\\d)',
|
||||
replacement: '[身份证]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['zh-CN'],
|
||||
},
|
||||
{
|
||||
id: 'cn_bank_card',
|
||||
label: 'desensitize.rules.cn_bank_card',
|
||||
pattern: '(?<!\\d)\\d{16,19}(?!\\d)',
|
||||
replacement: '[银行卡]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: ['zh-CN'],
|
||||
},
|
||||
{
|
||||
id: 'cn_landline',
|
||||
label: 'desensitize.rules.cn_landline',
|
||||
pattern: '(?<!\\d)0\\d{2,3}-?\\d{7,8}(?!\\d)',
|
||||
replacement: '[座机号]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: ['zh-CN'],
|
||||
},
|
||||
|
||||
// ==================== 美国 (en-US) ====================
|
||||
{
|
||||
id: 'us_ssn',
|
||||
label: 'desensitize.rules.us_ssn',
|
||||
pattern: '(?<!\\d)\\d{3}-\\d{2}-\\d{4}(?!\\d)',
|
||||
replacement: '[SSN]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['en-US'],
|
||||
},
|
||||
{
|
||||
id: 'us_phone',
|
||||
label: 'desensitize.rules.us_phone',
|
||||
pattern: '(?<!\\d)(?:\\+?1[-\\s.]?)?\\(?\\d{3}\\)?[-\\s.]?\\d{3}[-\\s.]?\\d{4}(?!\\d)',
|
||||
replacement: '[Phone]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['en-US'],
|
||||
},
|
||||
{
|
||||
id: 'us_drivers_license',
|
||||
label: 'desensitize.rules.us_drivers_license',
|
||||
pattern: '(?<![A-Z])\\b[A-Z]\\d{7,8}\\b',
|
||||
replacement: "[Driver's License]",
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: ['en-US'],
|
||||
},
|
||||
|
||||
// ==================== 日本 (ja-JP) ====================
|
||||
{
|
||||
id: 'jp_phone',
|
||||
label: 'desensitize.rules.jp_phone',
|
||||
pattern: '(?<!\\d)0[789]0-?\\d{4}-?\\d{4}(?!\\d)',
|
||||
replacement: '[電話番号]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['ja-JP'],
|
||||
},
|
||||
{
|
||||
id: 'jp_my_number',
|
||||
label: 'desensitize.rules.jp_my_number',
|
||||
pattern: '(?<!\\d)\\d{4}\\s?\\d{4}\\s?\\d{4}(?!\\d)',
|
||||
replacement: '[マイナンバー]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: ['ja-JP'],
|
||||
},
|
||||
|
||||
// ==================== 韩国 (ko-KR) ====================
|
||||
{
|
||||
id: 'kr_phone',
|
||||
label: 'desensitize.rules.kr_phone',
|
||||
pattern: '(?<!\\d)01[016789]-?\\d{3,4}-?\\d{4}(?!\\d)',
|
||||
replacement: '[전화번호]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['ko-KR'],
|
||||
},
|
||||
{
|
||||
id: 'kr_rrn',
|
||||
label: 'desensitize.rules.kr_rrn',
|
||||
pattern: '(?<!\\d)\\d{6}-[1-4]\\d{6}(?!\\d)',
|
||||
replacement: '[주민번호]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: ['ko-KR'],
|
||||
},
|
||||
|
||||
// ==================== 凭据 / Token (所有语言) ====================
|
||||
{
|
||||
id: 'api_key_prefix',
|
||||
label: 'desensitize.rules.api_key_prefix',
|
||||
pattern: '\\b(?:sk-|pk_(?:live|test)_|ghp_|gho_|ghs_|ghu_|glpat-|xoxb-|xoxp-|AKIA)[A-Za-z0-9_\\-]{10,}\\b',
|
||||
replacement: '[API Key]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
{
|
||||
id: 'bearer_token',
|
||||
label: 'desensitize.rules.bearer_token',
|
||||
pattern: 'Bearer\\s+[A-Za-z0-9\\-._~+/]+=*',
|
||||
replacement: 'Bearer [Token]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
|
||||
// ==================== 通用 (所有语言) ====================
|
||||
{
|
||||
id: 'email',
|
||||
label: 'desensitize.rules.email',
|
||||
pattern: '[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}',
|
||||
replacement: '[Email]',
|
||||
enabled: true,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
{
|
||||
id: 'credit_card',
|
||||
label: 'desensitize.rules.credit_card',
|
||||
pattern:
|
||||
'(?<!\\d)(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2}|6(?:011|5\\d{2}))[-\\s.]?\\d{4}[-\\s.]?\\d{4}[-\\s.]?\\d{4}(?!\\d)',
|
||||
replacement: '[Credit Card]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
{
|
||||
id: 'ipv4',
|
||||
label: 'desensitize.rules.ipv4',
|
||||
pattern:
|
||||
'(?<!\\d)(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)(?!\\d)',
|
||||
replacement: '[IP]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
{
|
||||
id: 'url',
|
||||
label: 'desensitize.rules.url',
|
||||
pattern: 'https?://[^\\s<>"]+',
|
||||
replacement: '[URL]',
|
||||
enabled: false,
|
||||
builtin: true,
|
||||
locales: [],
|
||||
},
|
||||
]
|
||||
|
||||
/**
|
||||
* 获取指定 locale 的默认规则（当前 locale 特定 + 通用规则）
|
||||
*/
|
||||
export function getDefaultRulesForLocale(locale: string): DesensitizeRule[] {
|
||||
return BUILTIN_DESENSITIZE_RULES.filter((rule) => rule.locales.length === 0 || rule.locales.includes(locale)).map(
|
||||
(rule) => ({ ...rule })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 合并新 locale 的规则到现有规则列表
|
||||
*/
|
||||
export function mergeRulesForLocale(existing: DesensitizeRule[], locale: string): DesensitizeRule[] {
|
||||
const existingIds = new Set(existing.map((r) => r.id))
|
||||
const newRules = BUILTIN_DESENSITIZE_RULES.filter(
|
||||
(rule) => !existingIds.has(rule.id) && (rule.locales.length === 0 || rule.locales.includes(locale))
|
||||
).map((rule) => ({ ...rule }))
|
||||
|
||||
return [...existing, ...newRules]
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* 工具结果格式化 & i18n 辅助（平台无关）
|
||||
*/
|
||||
|
||||
export function isChineseLocale(locale?: string): boolean {
|
||||
return locale?.startsWith('zh') ?? false
|
||||
}
|
||||
|
||||
export const i18nTexts = {
|
||||
allTime: { zh: '全部时间', en: 'All time' },
|
||||
noContent: { zh: '[无内容]', en: '[No content]' },
|
||||
memberNotFound: { zh: '未找到该成员', en: 'Member not found' },
|
||||
untilNow: { zh: '至今', en: 'Present' },
|
||||
noChangeRecord: { zh: '无变更记录', en: 'No change record' },
|
||||
noConversation: { zh: '未找到这两人之间的对话', en: 'No conversation found between these two members' },
|
||||
noMessageContext: { zh: '未找到指定的消息或上下文', en: 'Message or context not found' },
|
||||
messages: { zh: '条', en: '' },
|
||||
alias: { zh: '别名', en: 'Alias' },
|
||||
weekdays: {
|
||||
zh: ['', '周一', '周二', '周三', '周四', '周五', '周六', '周日'],
|
||||
en: ['', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'],
|
||||
},
|
||||
dailySummary: {
|
||||
zh: (days: number, total: number, avg: number) => `最近${days}天共${total}条，日均${avg}条`,
|
||||
en: (days: number, total: number, avg: number) => `Last ${days} days: ${total} messages, avg ${avg}/day`,
|
||||
},
|
||||
}
|
||||
|
||||
type TextEntryKey = Exclude<keyof typeof i18nTexts, 'dailySummary'>
|
||||
|
||||
export function t(key: TextEntryKey, locale?: string): string | string[] {
|
||||
const text = i18nTexts[key]
|
||||
if (typeof text === 'object' && 'zh' in text && 'en' in text) {
|
||||
return isChineseLocale(locale) ? text.zh : text.en
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
const MAX_MESSAGE_CONTENT_LENGTH = 200
|
||||
|
||||
/**
|
||||
* 格式化消息为简洁文本格式
|
||||
* 输出格式: "2025/3/3 07:25:04 张三: 消息内容"
|
||||
*/
|
||||
export function formatMessageCompact(
|
||||
msg: {
|
||||
id?: number
|
||||
senderName: string
|
||||
content: string | null
|
||||
timestamp: number
|
||||
},
|
||||
locale?: string
|
||||
): string {
|
||||
const localeStr = isChineseLocale(locale) ? 'zh-CN' : 'en-US'
|
||||
const time = new Date(msg.timestamp * 1000).toLocaleString(localeStr)
|
||||
let content = msg.content || (t('noContent', locale) as string)
|
||||
|
||||
if (content.length > MAX_MESSAGE_CONTENT_LENGTH) {
|
||||
content = content.slice(0, MAX_MESSAGE_CONTENT_LENGTH) + '...'
|
||||
}
|
||||
|
||||
return `${time} ${msg.senderName}: ${content}`
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化时间范围用于返回结果
|
||||
*/
|
||||
export function formatTimeRange(
|
||||
timeFilter?: { startTs: number; endTs: number },
|
||||
locale?: string
|
||||
): string | { start: string; end: string } {
|
||||
if (!timeFilter) return t('allTime', locale) as string
|
||||
const localeStr = isChineseLocale(locale) ? 'zh-CN' : 'en-US'
|
||||
return {
|
||||
start: new Date(timeFilter.startTs * 1000).toLocaleString(localeStr),
|
||||
end: new Date(timeFilter.endTs * 1000).toLocaleString(localeStr),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将工具返回的结构化数据格式化为 LLM 友好的纯文本
|
||||
*/
|
||||
export function formatToolResultAsText(details: Record<string, unknown>): string {
|
||||
const lines: string[] = []
|
||||
const messages = details.messages as string[] | undefined
|
||||
|
||||
for (const [key, value] of Object.entries(details)) {
|
||||
if (key === 'messages') continue
|
||||
if (value === undefined || value === null) continue
|
||||
|
||||
if (typeof value === 'object') {
|
||||
if ('start' in (value as Record<string, unknown>) && 'end' in (value as Record<string, unknown>)) {
|
||||
const range = value as { start: string; end: string }
|
||||
lines.push(`${key}: ${range.start} ~ ${range.end}`)
|
||||
} else if (Array.isArray(value)) {
|
||||
lines.push(`${key}: ${value.join(', ')}`)
|
||||
} else {
|
||||
lines.push(`${key}: ${JSON.stringify(value)}`)
|
||||
}
|
||||
} else {
|
||||
lines.push(`${key}: ${value}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (messages && messages.length > 0) {
|
||||
lines.push('')
|
||||
let lastDate = ''
|
||||
for (const msg of messages) {
|
||||
const spaceIdx = msg.indexOf(' ')
|
||||
const secondSpaceIdx = msg.indexOf(' ', spaceIdx + 1)
|
||||
if (spaceIdx > 0 && secondSpaceIdx > 0) {
|
||||
const date = msg.slice(0, spaceIdx)
|
||||
const rest = msg.slice(spaceIdx + 1)
|
||||
if (date !== lastDate) {
|
||||
lines.push(`--- ${date} ---`)
|
||||
lastDate = date
|
||||
}
|
||||
lines.push(rest)
|
||||
} else {
|
||||
lines.push(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* 昵称匿名化：用 U{senderId} 替代真实昵称
|
||||
* 就地修改 messages 的 senderName，返回映射表文本行
|
||||
*/
|
||||
export function anonymizeMessageNames(
|
||||
messages: Array<{ senderId?: number; senderName: string; senderPlatformId?: string }>,
|
||||
ownerPlatformId?: string
|
||||
): string {
|
||||
const nameMap = new Map<number, { name: string; platformId?: string }>()
|
||||
for (const msg of messages) {
|
||||
if (msg.senderId != null && !nameMap.has(msg.senderId)) {
|
||||
nameMap.set(msg.senderId, { name: msg.senderName, platformId: msg.senderPlatformId })
|
||||
}
|
||||
}
|
||||
|
||||
if (nameMap.size === 0) return ''
|
||||
|
||||
for (const msg of messages) {
|
||||
if (msg.senderId != null) {
|
||||
msg.senderName = `U${msg.senderId}`
|
||||
}
|
||||
}
|
||||
|
||||
const entries: string[] = []
|
||||
for (const [id, { name, platformId }] of nameMap) {
|
||||
const isOwner = ownerPlatformId && platformId === ownerPlatformId
|
||||
entries.push(`U${id}=${name}${isOwner ? '(owner)' : ''}`)
|
||||
}
|
||||
|
||||
return `[Name Map] ${entries.join(' | ')}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Token-aware 截断：在 token 预算内保留尽可能多的消息
|
||||
*/
|
||||
export function truncateFormattedMessages(
|
||||
formatted: string[],
|
||||
maxTokens: number,
|
||||
strategy: 'keep_first' | 'keep_last',
|
||||
countTokensFn: (text: string) => number
|
||||
): { messages: string[]; wasTruncated: boolean } {
|
||||
const budget = maxTokens - 200
|
||||
|
||||
let totalTokens = 0
|
||||
for (const line of formatted) {
|
||||
totalTokens += countTokensFn(line) + 1
|
||||
}
|
||||
if (totalTokens <= budget) {
|
||||
return { messages: formatted, wasTruncated: false }
|
||||
}
|
||||
|
||||
if (strategy === 'keep_first') {
|
||||
let tokens = 0
|
||||
let cutIndex = formatted.length
|
||||
for (let i = 0; i < formatted.length; i++) {
|
||||
tokens += countTokensFn(formatted[i]) + 1
|
||||
if (tokens > budget) {
|
||||
cutIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
return { messages: formatted.slice(0, cutIndex), wasTruncated: cutIndex < formatted.length }
|
||||
} else {
|
||||
let tokens = 0
|
||||
let cutIndex = 0
|
||||
for (let i = formatted.length - 1; i >= 0; i--) {
|
||||
tokens += countTokensFn(formatted[i]) + 1
|
||||
if (tokens > budget) {
|
||||
cutIndex = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
return { messages: formatted.slice(cutIndex), wasTruncated: cutIndex > 0 }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* 预处理管道（平台无关）
|
||||
*
|
||||
* 提供消息预处理、格式化、截断和脱敏功能。
|
||||
*/
|
||||
|
||||
export type { PreprocessConfig, PreprocessableMessage, DesensitizeRule, TruncationStrategy } from './types'
|
||||
export { preprocessMessages } from './pipeline'
|
||||
export type { PreprocessLogger } from './pipeline'
|
||||
export { BUILTIN_DESENSITIZE_RULES, getDefaultRulesForLocale, mergeRulesForLocale } from './builtin-rules'
|
||||
export {
|
||||
formatMessageCompact,
|
||||
formatTimeRange,
|
||||
formatToolResultAsText,
|
||||
anonymizeMessageNames,
|
||||
truncateFormattedMessages,
|
||||
isChineseLocale,
|
||||
i18nTexts,
|
||||
t,
|
||||
} from './format'
|
||||
@@ -0,0 +1,260 @@
|
||||
/**
|
||||
* 预处理管道
|
||||
* 执行顺序：数据清洗 → 黑名单 → 去噪 → 合并 → 脱敏
|
||||
*/
|
||||
|
||||
import type { PreprocessConfig, PreprocessableMessage, DesensitizeRule } from './types'
|
||||
|
||||
const MERGE_WINDOW_DEFAULT = 180
|
||||
|
||||
export interface PreprocessLogger {
|
||||
info(category: string, message: string, extra?: Record<string, unknown>): void
|
||||
warn(category: string, message: string, extra?: Record<string, unknown>): void
|
||||
}
|
||||
|
||||
const defaultLogger: PreprocessLogger = {
|
||||
info: () => {},
|
||||
warn: () => {},
|
||||
}
|
||||
|
||||
export function preprocessMessages<T extends PreprocessableMessage>(
|
||||
messages: T[],
|
||||
config?: PreprocessConfig,
|
||||
logger: PreprocessLogger = defaultLogger
|
||||
): T[] {
|
||||
if (!config || !hasAnyEnabled(config)) return messages
|
||||
if (messages.length === 0) return messages
|
||||
|
||||
const inputCount = messages.length
|
||||
let result: T[] = [...messages]
|
||||
const applied: string[] = []
|
||||
|
||||
if (config.dataCleaning !== false) {
|
||||
const cleaned = applyDataCleaning(result)
|
||||
if (cleaned.changed > 0) {
|
||||
result = cleaned.messages
|
||||
applied.push(`dataCleaning: ${cleaned.changed} messages cleaned`)
|
||||
}
|
||||
}
|
||||
|
||||
if (config.blacklistKeywords.length > 0) {
|
||||
const before = result.length
|
||||
result = applyBlacklistFilter(result, config.blacklistKeywords)
|
||||
applied.push(`blacklist: ${before} → ${result.length} (-${before - result.length})`)
|
||||
}
|
||||
|
||||
if (config.denoise) {
|
||||
const before = result.length
|
||||
result = applyDenoise(result)
|
||||
applied.push(`denoise: ${before} → ${result.length} (-${before - result.length})`)
|
||||
}
|
||||
|
||||
if (config.mergeConsecutive) {
|
||||
const before = result.length
|
||||
result = applyMergeConsecutive(result, config.mergeWindowSeconds ?? MERGE_WINDOW_DEFAULT)
|
||||
applied.push(`merge: ${before} → ${result.length} (-${before - result.length})`)
|
||||
}
|
||||
|
||||
if (config.desensitize) {
|
||||
const enabledRules = (config.desensitizeRules || []).filter((r) => r.enabled)
|
||||
if (enabledRules.length > 0) {
|
||||
result = applyDesensitize(result, enabledRules, logger)
|
||||
applied.push(`desensitize: ${enabledRules.length} rules applied`)
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Preprocess', `Pipeline: ${inputCount} → ${result.length} messages`, {
|
||||
strategies: applied,
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function hasAnyEnabled(config: PreprocessConfig): boolean {
|
||||
return (
|
||||
config.dataCleaning !== false ||
|
||||
config.mergeConsecutive ||
|
||||
config.blacklistKeywords.length > 0 ||
|
||||
config.denoise ||
|
||||
config.desensitize
|
||||
)
|
||||
}
|
||||
|
||||
// ==================== 策略实现 ====================
|
||||
|
||||
function applyDataCleaning<T extends PreprocessableMessage>(messages: T[]): { messages: T[]; changed: number } {
|
||||
let changed = 0
|
||||
const result = messages.map((msg) => {
|
||||
if (!msg.content) return msg
|
||||
const cleaned = cleanXmlContent(msg.content)
|
||||
if (cleaned === msg.content) return msg
|
||||
changed++
|
||||
return { ...msg, content: cleaned }
|
||||
})
|
||||
return { messages: result, changed }
|
||||
}
|
||||
|
||||
const XML_START = /^<\?xml\s|^<msg[\s>]/
|
||||
|
||||
function cleanXmlContent(content: string): string {
|
||||
const trimmed = content.trim()
|
||||
if (!XML_START.test(trimmed)) return content
|
||||
|
||||
const title = extractXmlTag(trimmed, 'title')
|
||||
const des = extractXmlTag(trimmed, 'des')
|
||||
|
||||
if (title) {
|
||||
return des ? `[分享] ${title} - ${des}` : `[分享] ${title}`
|
||||
}
|
||||
return '[应用消息]'
|
||||
}
|
||||
|
||||
function extractXmlTag(xml: string, tag: string): string | null {
|
||||
const openTag = `<${tag}>`
|
||||
const closeTag = `</${tag}>`
|
||||
const start = xml.indexOf(openTag)
|
||||
if (start === -1) return null
|
||||
const contentStart = start + openTag.length
|
||||
const end = xml.indexOf(closeTag, contentStart)
|
||||
if (end === -1) return null
|
||||
const value = xml.slice(contentStart, end).trim()
|
||||
return value.length > 0 ? value : null
|
||||
}
|
||||
|
||||
function applyBlacklistFilter<T extends PreprocessableMessage>(messages: T[], keywords: string[]): T[] {
|
||||
if (keywords.length === 0) return messages
|
||||
const lowerKeywords = keywords.map((k) => k.toLowerCase())
|
||||
return messages.filter((msg) => {
|
||||
if (!msg.content) return true
|
||||
const lower = msg.content.toLowerCase()
|
||||
return !lowerKeywords.some((kw) => lower.includes(kw))
|
||||
})
|
||||
}
|
||||
|
||||
function applyDenoise<T extends PreprocessableMessage>(messages: T[]): T[] {
|
||||
return messages.filter((msg) => {
|
||||
if (!msg.content) return false
|
||||
if (msg.replyToMessageId) return true
|
||||
const content = msg.content.trim()
|
||||
if (content.length === 0) return false
|
||||
if (content.length < 2) return false
|
||||
if (SYSTEM_PLACEHOLDERS.has(content)) return false
|
||||
if (isPureEmoji(content)) return false
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
const SYSTEM_PLACEHOLDERS = new Set([
|
||||
'[图片]',
|
||||
'[视频]',
|
||||
'[语音]',
|
||||
'[文件]',
|
||||
'[动画表情]',
|
||||
'[表情]',
|
||||
'[链接]',
|
||||
'[位置]',
|
||||
'[名片]',
|
||||
'[红包]',
|
||||
'[转账]',
|
||||
'[音乐]',
|
||||
'[Image]',
|
||||
'[Video]',
|
||||
'[Voice]',
|
||||
'[File]',
|
||||
'[Sticker]',
|
||||
'[Link]',
|
||||
])
|
||||
|
||||
function isPureEmoji(str: string): boolean {
|
||||
const stripped = str
|
||||
.replace(/\p{Emoji_Presentation}/gu, '')
|
||||
.replace(/\p{Extended_Pictographic}/gu, '')
|
||||
.replace(/\u200d/g, '')
|
||||
.replace(/\ufe0f/g, '')
|
||||
.replace(/\u20e3/g, '')
|
||||
.replace(/\s/g, '')
|
||||
return stripped.length === 0
|
||||
}
|
||||
|
||||
function applyMergeConsecutive<T extends PreprocessableMessage>(messages: T[], windowSeconds: number): T[] {
|
||||
if (messages.length <= 1) return messages
|
||||
|
||||
const merged: T[] = []
|
||||
let current: T | null = null
|
||||
|
||||
for (const msg of messages) {
|
||||
if (!current) {
|
||||
current = { ...msg }
|
||||
continue
|
||||
}
|
||||
|
||||
const sameSender = isSameSender(current, msg)
|
||||
const withinWindow = Math.abs(msg.timestamp - current.timestamp) <= windowSeconds
|
||||
|
||||
if (sameSender && withinWindow) {
|
||||
current = {
|
||||
...current,
|
||||
content: [current.content, msg.content].filter(Boolean).join('\n'),
|
||||
}
|
||||
} else {
|
||||
merged.push(current)
|
||||
current = { ...msg }
|
||||
}
|
||||
}
|
||||
|
||||
if (current) merged.push(current)
|
||||
return merged
|
||||
}
|
||||
|
||||
function isSameSender(a: PreprocessableMessage, b: PreprocessableMessage): boolean {
|
||||
if (a.senderPlatformId && b.senderPlatformId) {
|
||||
return a.senderPlatformId === b.senderPlatformId
|
||||
}
|
||||
return a.senderName === b.senderName
|
||||
}
|
||||
|
||||
function applyDesensitize<T extends PreprocessableMessage>(
|
||||
messages: T[],
|
||||
rules: DesensitizeRule[],
|
||||
logger: PreprocessLogger
|
||||
): T[] {
|
||||
const compiledRules = compileRules(rules, logger)
|
||||
if (compiledRules.length === 0) return messages
|
||||
|
||||
return messages.map((msg) => {
|
||||
if (!msg.content) return msg
|
||||
let content = msg.content
|
||||
for (const { regex, replacement } of compiledRules) {
|
||||
regex.lastIndex = 0
|
||||
content = content.replace(regex, replacement)
|
||||
}
|
||||
if (content === msg.content) return msg
|
||||
return { ...msg, content }
|
||||
})
|
||||
}
|
||||
|
||||
const regexCache = new Map<string, RegExp | null>()
|
||||
|
||||
function compileRules(
|
||||
rules: DesensitizeRule[],
|
||||
logger: PreprocessLogger
|
||||
): Array<{ regex: RegExp; replacement: string }> {
|
||||
const result: Array<{ regex: RegExp; replacement: string }> = []
|
||||
for (const rule of rules) {
|
||||
let regex = regexCache.get(rule.pattern)
|
||||
if (regex === undefined) {
|
||||
try {
|
||||
regex = new RegExp(rule.pattern, 'g')
|
||||
regexCache.set(rule.pattern, regex)
|
||||
} catch {
|
||||
logger.warn('Preprocess', `Invalid regex in desensitize rule "${rule.id}": ${rule.pattern}`)
|
||||
regexCache.set(rule.pattern, null)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if (regex) {
|
||||
result.push({ regex, replacement: rule.replacement })
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/** 单条脱敏规则 */
|
||||
export interface DesensitizeRule {
|
||||
/** 唯一标识（预置规则用固定 id，自定义规则用 uuid） */
|
||||
id: string
|
||||
/** 显示名称 */
|
||||
label: string
|
||||
/** 正则表达式字符串（运行时 new RegExp(pattern, 'g')） */
|
||||
pattern: string
|
||||
/** 替换文本 */
|
||||
replacement: string
|
||||
/** 是否启用 */
|
||||
enabled: boolean
|
||||
/** 是否为预置规则（预置规则不可删除，仅可启用/禁用） */
|
||||
builtin: boolean
|
||||
/** 适用的 locale 列表（空数组表示通用） */
|
||||
locales: string[]
|
||||
}
|
||||
|
||||
/** 预处理配置 */
|
||||
export interface PreprocessConfig {
|
||||
/** 数据清洗：清理 XML 卡片消息等非纯文本内容（默认开启） */
|
||||
dataCleaning: boolean
|
||||
/** 合并连续发言（同发送者 + 时间间隔 < mergeWindowSeconds） */
|
||||
mergeConsecutive: boolean
|
||||
/** 合并窗口（秒），默认 180 */
|
||||
mergeWindowSeconds?: number
|
||||
/** 自定义黑名单关键词，包含任一关键词的消息将被整条过滤 */
|
||||
blacklistKeywords: string[]
|
||||
/** 智能去噪（过滤纯语气词、纯表情、系统占位符） */
|
||||
denoise: boolean
|
||||
/** 数据脱敏总开关 */
|
||||
desensitize: boolean
|
||||
/** 脱敏规则列表（预置 + 自定义，按优先级排序） */
|
||||
desensitizeRules: DesensitizeRule[]
|
||||
/** 昵称匿名化：用 U{id} 替代真实昵称，减少 AI 幻觉 */
|
||||
anonymizeNames: boolean
|
||||
}
|
||||
|
||||
/** 预处理管道可接受的消息结构 */
|
||||
export interface PreprocessableMessage {
|
||||
id?: number
|
||||
senderId?: number
|
||||
senderName: string
|
||||
senderPlatformId?: string
|
||||
content: string | null
|
||||
timestamp: number
|
||||
replyToMessageId?: string | null
|
||||
}
|
||||
|
||||
export type TruncationStrategy = 'keep_first' | 'keep_last'
|
||||
@@ -35,6 +35,30 @@ export type { AssistantConfig, AssistantSummary, SkillDef, SkillSummary } from '
|
||||
export { parseAssistantFile, serializeAssistant, parseSkillFile, extractSkillId } from './ai'
|
||||
export { AIConversationManager } from './ai'
|
||||
export { countTokens, countMessagesTokens } from './ai'
|
||||
|
||||
// Preprocessor
|
||||
export type {
|
||||
PreprocessConfig,
|
||||
PreprocessableMessage,
|
||||
DesensitizeRule,
|
||||
TruncationStrategy,
|
||||
PreprocessLogger,
|
||||
} from './ai'
|
||||
export {
|
||||
preprocessMessages,
|
||||
BUILTIN_DESENSITIZE_RULES,
|
||||
getDefaultRulesForLocale,
|
||||
mergeRulesForLocale,
|
||||
formatMessageCompact,
|
||||
formatTimeRange,
|
||||
formatToolResultAsText,
|
||||
anonymizeMessageNames,
|
||||
truncateFormattedMessages,
|
||||
isChineseLocale,
|
||||
i18nTexts,
|
||||
t,
|
||||
} from './ai'
|
||||
|
||||
export type {
|
||||
AIConversation,
|
||||
AIMessage,
|
||||
|
||||
Reference in new issue
Block a user