fix(semantic-index): harden embedding input guards

packages/node-runtime/src/semantic-index/chunker.test.ts

@@ -132,6 +132,86 @@ test('single oversized message is clamped to the child token hard limit', () =>
   assert.ok(estimateTokens(result.chunks[0].embeddingInput) <= cfg.childHardMaxTokens)
 })
 
+test('single oversized non-CJK input also respects the UTF-8 byte safety ceiling', () => {
+  const cfg: ChunkerConfig = {
+    ...DEFAULT_CHUNKER_CONFIG,
+    parentGapSeconds: 100000,
+    parentMaxTokens: 100000,
+    childTargetMaxChars: 100000,
+    childHardMaxTokens: 120,
+    semanticVoidSkipThreshold: 1,
+  }
+  const result = chunkMessages({
+    messages: [msg(1, '😀'.repeat(1000), 0)],
+    source: privateSource,
+    config: cfg,
+  })
+
+  const embeddingInput = result.chunks[0].embeddingInput
+  assert.ok(Buffer.byteLength(embeddingInput, 'utf8') <= cfg.childHardMaxTokens * 4)
+})
+
+test('oversized source title cannot consume the message body budget', () => {
+  const cfg: ChunkerConfig = {
+    ...DEFAULT_CHUNKER_CONFIG,
+    parentGapSeconds: 100000,
+    parentMaxTokens: 100000,
+    childTargetMaxChars: 100000,
+    childHardMaxTokens: 120,
+    semanticVoidSkipThreshold: 1,
+  }
+  const marker = '正文唯一标记内容'
+  const result = chunkMessages({
+    messages: [msg(1, marker.repeat(10), 0)],
+    source: { title: '群'.repeat(1000), kind: 'private' },
+    config: cfg,
+  })
+
+  assert.ok(result.chunks[0].embeddingInput.includes(marker))
+})
+
+test('oversized sender name cannot consume the message content budget', () => {
+  const cfg: ChunkerConfig = {
+    ...DEFAULT_CHUNKER_CONFIG,
+    parentGapSeconds: 100000,
+    parentMaxTokens: 100000,
+    childTargetMaxChars: 100000,
+    childHardMaxTokens: 120,
+    semanticVoidSkipThreshold: 1,
+  }
+  const marker = '消息正文唯一标记'
+  const result = chunkMessages({
+    messages: [msg(1, marker.repeat(10), 0, '成员'.repeat(1000))],
+    source: privateSource,
+    config: cfg,
+  })
+
+  assert.ok(result.chunks[0].embeddingInput.includes(marker))
+})
+
+test('repeated oversized sender names share one metadata budget across the body', () => {
+  const cfg: ChunkerConfig = {
+    ...DEFAULT_CHUNKER_CONFIG,
+    parentGapSeconds: 100000,
+    parentMaxTokens: 100000,
+    childTargetMinChars: 100000,
+    childTargetMaxChars: 100000,
+    childSoftMaxMessages: 100,
+    childHardMaxMessages: 100,
+    childHardMaxTokens: 1200,
+    overlapMessages: 0,
+    semanticVoidSkipThreshold: 1,
+  }
+  const marker = '末条消息正文唯一标记'
+  const messages = Array.from({ length: 20 }, (_, index) =>
+    msg(index + 1, index === 19 ? marker : `前置正文${index}`, 0, '成员'.repeat(1000))
+  )
+  const result = chunkMessages({ messages, source: privateSource, config: cfg })
+
+  assert.equal(result.chunks.length, 1)
+  assert.ok(result.chunks[0].embeddingInput.includes(marker))
+})
+
 test('semantic void messages are excluded from embedding body, header and effective chars', () => {
   const messages = [
     msg(1, '我们讨论一下下周的装修进度安排', 0, '张三'),

packages/node-runtime/src/semantic-index/chunker.ts

@@ -183,6 +183,10 @@ function sha256Hex(input: string): string {
 
 // ==================== header / body ====================
 
+const EMBEDDING_HEADER_BUDGET_RATIO = 0.25
+const SOURCE_TITLE_HEADER_SHARE = 1 / 3
+const SENDER_NAMES_BODY_BUDGET_RATIO = 0.1
+
 function pad2(n: number): string {
   return String(n).padStart(2, '0')
 }
@@ -207,24 +211,45 @@ function formatTimeRange(startTs: number, endTs: number): string {
   return `${formatDateTime(startTs)} - ${sameDay ? formatTime(endTs) : formatDateTime(endTs)}`
 }
 
-function buildEmbeddingInput(source: ChunkSource, messages: ChunkMessageInput[]): string {
+/**
+ * 在 token/字节总护栏内组装 embedding 文本。
+ * header 最多占四分之一预算，异常长的来源名或发送者名只截断派生文本，正文始终优先保留。
+ */
+function buildEmbeddingInput(source: ChunkSource, messages: ChunkMessageInput[], maxTokens: number): string {
+  const limit = Math.max(0, Math.floor(maxTokens))
+  if (limit === 0) return ''
+
   const effective = messages.filter((m) => !isSemanticVoid(m))
   const startTs = messages[0].ts
   const endTs = messages[messages.length - 1].ts
 
-  const lines: string[] = []
+  const headerLimit = Math.max(1, Math.floor(limit * EMBEDDING_HEADER_BUDGET_RATIO))
+  const bodyFloor = Math.max(1, limit - headerLimit)
+  const sourceTitleLimit = Math.max(1, Math.floor(headerLimit * SOURCE_TITLE_HEADER_SHARE))
+  const senderNamesTotalLimit = Math.floor(bodyFloor * SENDER_NAMES_BODY_BUDGET_RATIO)
+  const bodySenderNameLimit = effective.length > 0 ? Math.floor(senderNamesTotalLimit / effective.length) : 0
+  const clampHeaderSenderName = (name: string) => clampEstimatedTokens(name, sourceTitleLimit)
+  // 所有发送者名共享正文总预算，按有效消息数均分，避免多条异常长名称累计挤掉后半段正文。
+  const clampBodySenderName = (name: string) => clampEstimatedTokens(name, bodySenderNameLimit)
+
+  const headerLines: string[] = []
   const kindLabel = source.kind === 'group' ? '群聊' : '私聊'
-  lines.push(`[来源] ${source.title}（${kindLabel}）`)
-  lines.push(`[时间范围] ${formatTimeRange(startTs, endTs)}`)
+  headerLines.push(`[来源] ${clampEstimatedTokens(source.title, sourceTitleLimit)}（${kindLabel}）`)
+  headerLines.push(`[时间范围] ${formatTimeRange(startTs, endTs)}`)
   if (source.kind === 'group') {
-    const participants = [...new Set(effective.map((m) => m.senderName))]
-    lines.push(`[参与者] ${participants.join('、')}`)
+    const participants = [...new Set(effective.map((m) => clampHeaderSenderName(m.senderName)))]
+    headerLines.push(`[参与者] ${participants.join('、')}`)
   }
-  lines.push('')
-  for (const m of effective) {
-    lines.push(`${m.senderName}: ${(m.content ?? '').trim()}`)
-  }
-  return lines.join('\n')
+  const header = clampEstimatedTokens(headerLines.join('\n'), headerLimit)
+
+  const separator = '\n\n'
+  const remaining = Math.max(0, limit - estimateTokens(header) - estimateTokens(separator))
+  const body = clampEstimatedTokens(
+    effective.map((m) => `${clampBodySenderName(m.senderName)}: ${(m.content ?? '').trim()}`).join('\n'),
+    remaining
+  )
+  const combined = body ? `${header}${separator}${body}` : header
+  return clampEstimatedTokens(combined, limit)
 }
 
 function rawContentHashOf(messages: ChunkMessageInput[]): string {
@@ -316,7 +341,7 @@ export function chunkMessages(input: ChunkMessagesInput): ChunkResult {
       const startMessageId = draft[0].id
       const endMessageId = draft[draft.length - 1].id
       // 单条超长消息无法在消息边界上继续切分，因此在派生的 embedding 文本上执行最终硬限制。
-      const embeddingInput = clampEstimatedTokens(buildEmbeddingInput(input.source, draft), config.childHardMaxTokens)
+      const embeddingInput = buildEmbeddingInput(input.source, draft, config.childHardMaxTokens)
       chunks.push({
         localChunkId: `${parentId}#${startMessageId}-${endMessageId}`,
         parentId,

packages/node-runtime/src/semantic-index/tokens.ts

@@ -4,6 +4,10 @@
  * 仅作硬上限护栏与证据预算控制，真实 token 由 embedder / LLM 决定。
  * 估算规则：CJK 每字约 1 token，其余字符约 1/4 token。
  */
+import { Buffer } from 'node:buffer'
+
+const UTF8_BYTES_PER_ESTIMATED_TOKEN = 4
+
 function isCjkCharacter(char: string): boolean {
   return /[\u3400-\u9fff\uf900-\ufaff]/u.test(char)
 }
@@ -19,21 +23,28 @@ export function estimateTokens(text: string): number {
 }
 
 /**
- * 按共享估算规则截断文本，确保单条超长消息不会绕过 chunk token 硬上限。
+ * 按共享估算规则与 UTF-8 字节上限截断文本，避免 emoji 等高膨胀字符绕过安全护栏。
  * 这里只处理送给 embedding provider 的派生文本，不修改原始消息和证据范围。
  */
 export function clampEstimatedTokens(text: string, maxTokens: number): string {
   const limit = Math.max(0, Math.floor(maxTokens))
   if (limit === 0) return ''
-  if (estimateTokens(text) <= limit) return text
+  const byteLimit = limit * UTF8_BYTES_PER_ESTIMATED_TOKEN
+  if (estimateTokens(text) <= limit && Buffer.byteLength(text, 'utf8') <= byteLimit) return text
 
   let cjk = 0
   let other = 0
+  let bytes = 0
   let end = 0
   for (const char of text) {
-    if (isCjkCharacter(char)) cjk++
-    else other++
-    if (cjk + Math.ceil(other / 4) > limit) break
+    const isCjk = isCjkCharacter(char)
+    const nextCjk = cjk + (isCjk ? 1 : 0)
+    const nextOther = other + (isCjk ? 0 : 1)
+    const nextBytes = bytes + Buffer.byteLength(char, 'utf8')
+    if (nextCjk + Math.ceil(nextOther / 4) > limit || nextBytes > byteLimit) break
+    cjk = nextCjk
+    other = nextOther
+    bytes = nextBytes
     end += char.length
   }
   return text.slice(0, end)