diff --git a/packages/node-runtime/src/semantic-index/chunker.test.ts b/packages/node-runtime/src/semantic-index/chunker.test.ts index 96b455ad..7a90db6a 100644 --- a/packages/node-runtime/src/semantic-index/chunker.test.ts +++ b/packages/node-runtime/src/semantic-index/chunker.test.ts @@ -132,6 +132,86 @@ test('single oversized message is clamped to the child token hard limit', () => assert.ok(estimateTokens(result.chunks[0].embeddingInput) <= cfg.childHardMaxTokens) }) +test('single oversized non-CJK input also respects the UTF-8 byte safety ceiling', () => { + const cfg: ChunkerConfig = { + ...DEFAULT_CHUNKER_CONFIG, + parentGapSeconds: 100000, + parentMaxTokens: 100000, + childTargetMaxChars: 100000, + childHardMaxTokens: 120, + semanticVoidSkipThreshold: 1, + } + const result = chunkMessages({ + messages: [msg(1, '😀'.repeat(1000), 0)], + source: privateSource, + config: cfg, + }) + + const embeddingInput = result.chunks[0].embeddingInput + assert.ok(Buffer.byteLength(embeddingInput, 'utf8') <= cfg.childHardMaxTokens * 4) +}) + +test('oversized source title cannot consume the message body budget', () => { + const cfg: ChunkerConfig = { + ...DEFAULT_CHUNKER_CONFIG, + parentGapSeconds: 100000, + parentMaxTokens: 100000, + childTargetMaxChars: 100000, + childHardMaxTokens: 120, + semanticVoidSkipThreshold: 1, + } + const marker = '正文唯一标记内容' + const result = chunkMessages({ + messages: [msg(1, marker.repeat(10), 0)], + source: { title: '群'.repeat(1000), kind: 'private' }, + config: cfg, + }) + + assert.ok(result.chunks[0].embeddingInput.includes(marker)) +}) + +test('oversized sender name cannot consume the message content budget', () => { + const cfg: ChunkerConfig = { + ...DEFAULT_CHUNKER_CONFIG, + parentGapSeconds: 100000, + parentMaxTokens: 100000, + childTargetMaxChars: 100000, + childHardMaxTokens: 120, + semanticVoidSkipThreshold: 1, + } + const marker = '消息正文唯一标记' + const result = chunkMessages({ + messages: [msg(1, marker.repeat(10), 0, '成员'.repeat(1000))], + source: privateSource, + config: cfg, + }) + + assert.ok(result.chunks[0].embeddingInput.includes(marker)) +}) + +test('repeated oversized sender names share one metadata budget across the body', () => { + const cfg: ChunkerConfig = { + ...DEFAULT_CHUNKER_CONFIG, + parentGapSeconds: 100000, + parentMaxTokens: 100000, + childTargetMinChars: 100000, + childTargetMaxChars: 100000, + childSoftMaxMessages: 100, + childHardMaxMessages: 100, + childHardMaxTokens: 1200, + overlapMessages: 0, + semanticVoidSkipThreshold: 1, + } + const marker = '末条消息正文唯一标记' + const messages = Array.from({ length: 20 }, (_, index) => + msg(index + 1, index === 19 ? marker : `前置正文${index}`, 0, '成员'.repeat(1000)) + ) + const result = chunkMessages({ messages, source: privateSource, config: cfg }) + + assert.equal(result.chunks.length, 1) + assert.ok(result.chunks[0].embeddingInput.includes(marker)) +}) + test('semantic void messages are excluded from embedding body, header and effective chars', () => { const messages = [ msg(1, '我们讨论一下下周的装修进度安排', 0, '张三'), diff --git a/packages/node-runtime/src/semantic-index/chunker.ts b/packages/node-runtime/src/semantic-index/chunker.ts index 7fc381cb..a55e3d7c 100644 --- a/packages/node-runtime/src/semantic-index/chunker.ts +++ b/packages/node-runtime/src/semantic-index/chunker.ts @@ -183,6 +183,10 @@ function sha256Hex(input: string): string { // ==================== header / body ==================== +const EMBEDDING_HEADER_BUDGET_RATIO = 0.25 +const SOURCE_TITLE_HEADER_SHARE = 1 / 3 +const SENDER_NAMES_BODY_BUDGET_RATIO = 0.1 + function pad2(n: number): string { return String(n).padStart(2, '0') } @@ -207,24 +211,45 @@ function formatTimeRange(startTs: number, endTs: number): string { return `${formatDateTime(startTs)} - ${sameDay ? formatTime(endTs) : formatDateTime(endTs)}` } -function buildEmbeddingInput(source: ChunkSource, messages: ChunkMessageInput[]): string { +/** + * 在 token/字节总护栏内组装 embedding 文本。 + * header 最多占四分之一预算，异常长的来源名或发送者名只截断派生文本，正文始终优先保留。 + */ +function buildEmbeddingInput(source: ChunkSource, messages: ChunkMessageInput[], maxTokens: number): string { + const limit = Math.max(0, Math.floor(maxTokens)) + if (limit === 0) return '' + const effective = messages.filter((m) => !isSemanticVoid(m)) const startTs = messages[0].ts const endTs = messages[messages.length - 1].ts - const lines: string[] = [] + const headerLimit = Math.max(1, Math.floor(limit * EMBEDDING_HEADER_BUDGET_RATIO)) + const bodyFloor = Math.max(1, limit - headerLimit) + const sourceTitleLimit = Math.max(1, Math.floor(headerLimit * SOURCE_TITLE_HEADER_SHARE)) + const senderNamesTotalLimit = Math.floor(bodyFloor * SENDER_NAMES_BODY_BUDGET_RATIO) + const bodySenderNameLimit = effective.length > 0 ? Math.floor(senderNamesTotalLimit / effective.length) : 0 + const clampHeaderSenderName = (name: string) => clampEstimatedTokens(name, sourceTitleLimit) + // 所有发送者名共享正文总预算，按有效消息数均分，避免多条异常长名称累计挤掉后半段正文。 + const clampBodySenderName = (name: string) => clampEstimatedTokens(name, bodySenderNameLimit) + + const headerLines: string[] = [] const kindLabel = source.kind === 'group' ? '群聊' : '私聊' - lines.push(`[来源] ${source.title}（${kindLabel}）`) - lines.push(`[时间范围] ${formatTimeRange(startTs, endTs)}`) + headerLines.push(`[来源] ${clampEstimatedTokens(source.title, sourceTitleLimit)}（${kindLabel}）`) + headerLines.push(`[时间范围] ${formatTimeRange(startTs, endTs)}`) if (source.kind === 'group') { - const participants = [...new Set(effective.map((m) => m.senderName))] - lines.push(`[参与者] ${participants.join('、')}`) + const participants = [...new Set(effective.map((m) => clampHeaderSenderName(m.senderName)))] + headerLines.push(`[参与者] ${participants.join('、')}`) } - lines.push('') - for (const m of effective) { - lines.push(`${m.senderName}: ${(m.content ?? '').trim()}`) - } - return lines.join('\n') + const header = clampEstimatedTokens(headerLines.join('\n'), headerLimit) + + const separator = '\n\n' + const remaining = Math.max(0, limit - estimateTokens(header) - estimateTokens(separator)) + const body = clampEstimatedTokens( + effective.map((m) => `${clampBodySenderName(m.senderName)}: ${(m.content ?? '').trim()}`).join('\n'), + remaining + ) + const combined = body ? `${header}${separator}${body}` : header + return clampEstimatedTokens(combined, limit) } function rawContentHashOf(messages: ChunkMessageInput[]): string { @@ -316,7 +341,7 @@ export function chunkMessages(input: ChunkMessagesInput): ChunkResult { const startMessageId = draft[0].id const endMessageId = draft[draft.length - 1].id // 单条超长消息无法在消息边界上继续切分，因此在派生的 embedding 文本上执行最终硬限制。 - const embeddingInput = clampEstimatedTokens(buildEmbeddingInput(input.source, draft), config.childHardMaxTokens) + const embeddingInput = buildEmbeddingInput(input.source, draft, config.childHardMaxTokens) chunks.push({ localChunkId: `${parentId}#${startMessageId}-${endMessageId}`, parentId, diff --git a/packages/node-runtime/src/semantic-index/tokens.ts b/packages/node-runtime/src/semantic-index/tokens.ts index bd1a7434..0f20f02a 100644 --- a/packages/node-runtime/src/semantic-index/tokens.ts +++ b/packages/node-runtime/src/semantic-index/tokens.ts @@ -4,6 +4,10 @@ * 仅作硬上限护栏与证据预算控制，真实 token 由 embedder / LLM 决定。 * 估算规则：CJK 每字约 1 token，其余字符约 1/4 token。 */ +import { Buffer } from 'node:buffer' + +const UTF8_BYTES_PER_ESTIMATED_TOKEN = 4 + function isCjkCharacter(char: string): boolean { return /[\u3400-\u9fff\uf900-\ufaff]/u.test(char) } @@ -19,21 +23,28 @@ export function estimateTokens(text: string): number { } /** - * 按共享估算规则截断文本，确保单条超长消息不会绕过 chunk token 硬上限。 + * 按共享估算规则与 UTF-8 字节上限截断文本，避免 emoji 等高膨胀字符绕过安全护栏。 * 这里只处理送给 embedding provider 的派生文本，不修改原始消息和证据范围。 */ export function clampEstimatedTokens(text: string, maxTokens: number): string { const limit = Math.max(0, Math.floor(maxTokens)) if (limit === 0) return '' - if (estimateTokens(text) <= limit) return text + const byteLimit = limit * UTF8_BYTES_PER_ESTIMATED_TOKEN + if (estimateTokens(text) <= limit && Buffer.byteLength(text, 'utf8') <= byteLimit) return text let cjk = 0 let other = 0 + let bytes = 0 let end = 0 for (const char of text) { - if (isCjkCharacter(char)) cjk++ - else other++ - if (cjk + Math.ceil(other / 4) > limit) break + const isCjk = isCjkCharacter(char) + const nextCjk = cjk + (isCjk ? 1 : 0) + const nextOther = other + (isCjk ? 0 : 1) + const nextBytes = bytes + Buffer.byteLength(char, 'utf8') + if (nextCjk + Math.ceil(nextOther / 4) > limit || nextBytes > byteLimit) break + cjk = nextCjk + other = nextOther + bytes = nextBytes end += char.length } return text.slice(0, end)