test(ai): add agent routing evaluation set

packages/node-runtime/src/ai/agent/__tests__/evaluation-set.test.ts

@@ -0,0 +1,43 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { AI_AGENT_ROUTING_EVALUATION_SET, REQUIRED_EVALUATION_SCENARIOS } from '../evaluation-set'
+
+describe('AI agent routing evaluation set', () => {
+  it('keeps a fixed phase-one sample size with stable unique ids', () => {
+    assert.ok(AI_AGENT_ROUTING_EVALUATION_SET.length >= 20)
+    assert.ok(AI_AGENT_ROUTING_EVALUATION_SET.length <= 30)
+
+    const ids = new Set(AI_AGENT_ROUTING_EVALUATION_SET.map((item) => item.id))
+    assert.equal(ids.size, AI_AGENT_ROUTING_EVALUATION_SET.length)
+    for (const item of AI_AGENT_ROUTING_EVALUATION_SET) {
+      assert.match(item.id, /^ai-route-eval-\d{3}$/)
+    }
+  })
+
+  it('covers every expected route and required scenario', () => {
+    const routes = new Set(AI_AGENT_ROUTING_EVALUATION_SET.map((item) => item.expectedRoute))
+    assert.deepEqual(routes, new Set(['direct_response', 'tool_assisted', 'planned_execution']))
+
+    const scenarios = new Set(AI_AGENT_ROUTING_EVALUATION_SET.flatMap((item) => item.scenarios))
+    for (const scenario of REQUIRED_EVALUATION_SCENARIOS) {
+      assert.ok(scenarios.has(scenario), `missing scenario: ${scenario}`)
+    }
+  })
+
+  it('requires evidence coverage points for planned execution cases only', () => {
+    for (const item of AI_AGENT_ROUTING_EVALUATION_SET) {
+      if (item.expectedRoute === 'planned_execution') {
+        assert.ok(item.expectedEvidenceCoverage.length >= 2, `${item.id} should define evidence coverage`)
+      } else {
+        assert.equal(item.expectedEvidenceCoverage.length, 0, `${item.id} should not define evidence coverage`)
+      }
+    }
+  })
+
+  it('marks baseline collection as pending real-environment execution', () => {
+    for (const item of AI_AGENT_ROUTING_EVALUATION_SET) {
+      assert.equal(item.baseline.status, 'pending_real_environment_run')
+      assert.ok(item.baseline.notes.length > 0)
+    }
+  })
+})

packages/node-runtime/src/ai/agent/evaluation-set.ts

@@ -0,0 +1,342 @@
+export type RequestRoute = 'direct_response' | 'tool_assisted' | 'planned_execution'
+
+export type EvaluationScenario =
+  | 'casual_chat'
+  | 'concept_explanation'
+  | 'help_or_configuration'
+  | 'simple_data_query'
+  | 'simple_search'
+  | 'long_range_trend_analysis'
+  | 'key_member_influence'
+  | 'relationship_analysis'
+  | 'topic_evolution'
+  | 'search_failure_recovery'
+  | 'multi_condition_filter'
+  | 'member_lookup_then_messages'
+  | 'insufficient_evidence'
+
+export interface RouteEvaluationCase {
+  id: string
+  locale: 'zh-CN' | 'en-US'
+  chatType: 'group' | 'private'
+  userMessage: string
+  expectedRoute: RequestRoute
+  scenarios: EvaluationScenario[]
+  expectedEvidenceCoverage: string[]
+  baseline: {
+    status: 'pending_real_environment_run'
+    notes: string
+  }
+}
+
+export const REQUIRED_EVALUATION_SCENARIOS: readonly EvaluationScenario[] = [
+  'casual_chat',
+  'concept_explanation',
+  'help_or_configuration',
+  'simple_data_query',
+  'simple_search',
+  'long_range_trend_analysis',
+  'key_member_influence',
+  'relationship_analysis',
+  'topic_evolution',
+  'search_failure_recovery',
+  'multi_condition_filter',
+  'member_lookup_then_messages',
+  'insufficient_evidence',
+]
+
+const pendingBaseline = {
+  status: 'pending_real_environment_run',
+  notes: 'Requires a configured LLM service and a real or fixture chat database; do not fake baseline results.',
+} as const
+
+export const AI_AGENT_ROUTING_EVALUATION_SET: readonly RouteEvaluationCase[] = [
+  {
+    id: 'ai-route-eval-001',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '你觉得我今天应该先整理资料还是先回复消息？',
+    expectedRoute: 'direct_response',
+    scenarios: ['casual_chat'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-002',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '解释一下什么是 Function Calling Agent，和 ReACT 有什么区别？',
+    expectedRoute: 'direct_response',
+    scenarios: ['concept_explanation'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-003',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: 'ChatLab 的 AI 日志在哪里看？',
+    expectedRoute: 'direct_response',
+    scenarios: ['help_or_configuration'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-004',
+    locale: 'zh-CN',
+    chatType: 'private',
+    userMessage: '不用查聊天记录，帮我把这句话润色得自然一点：明天我可能晚点到。',
+    expectedRoute: 'direct_response',
+    scenarios: ['casual_chat'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-005',
+    locale: 'en-US',
+    chatType: 'group',
+    userMessage: 'Briefly explain why native function calling is safer than text-formatted actions.',
+    expectedRoute: 'direct_response',
+    scenarios: ['concept_explanation'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-006',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '帮我看看这个群一共有多少成员、多少条消息。',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_data_query'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-007',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '昨天大家主要聊了什么？简单概括一下。',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_search'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-008',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '找一下最近提到“报销”的聊天记录。',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_search'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-009',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '谁发言最多？给我前 5 名就行。',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_data_query'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-010',
+    locale: 'zh-CN',
+    chatType: 'private',
+    userMessage: '我和对方上周有没有聊到“面试”？',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_search'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-011',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '找一下昵称里带“小王”的成员，然后看他最近 20 条发言。',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['member_lookup_then_messages'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-012',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '这个群今天凌晨 0 点到 6 点有没有异常活跃？',
+    expectedRoute: 'tool_assisted',
+    scenarios: ['simple_data_query'],
+    expectedEvidenceCoverage: [],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-013',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '分析过去一年群里话题的变化趋势，按季度总结主要变化，并举出证据。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['long_range_trend_analysis', 'topic_evolution'],
+    expectedEvidenceCoverage: [
+      'Collect representative messages or summaries across at least three quarterly windows.',
+      'Compare topic distribution changes over time rather than summarizing one period only.',
+      'Cite concrete message examples or retrieved evidence for each major trend.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-014',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '找出这个群里影响力最强的 3 个人，并说明他们分别影响了哪些话题或互动。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['key_member_influence'],
+    expectedEvidenceCoverage: [
+      'Use member activity or interaction statistics to identify candidates.',
+      'Verify each candidate with message examples or conversation context.',
+      'Separate message volume from influence and state limitations.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-015',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '分析 Alice 和 Bob 最近半年的互动关系：谁更主动、回应是否及时、主要聊什么。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['relationship_analysis', 'multi_condition_filter'],
+    expectedEvidenceCoverage: [
+      'Resolve both members before querying their interaction.',
+      'Compare initiation, response timing, and topic evidence.',
+      'Use a bounded time range and cite representative exchanges.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-016',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '复盘今年项目相关讨论：从需求、开发、上线到反馈，每个阶段有哪些关键分歧？',
+    expectedRoute: 'planned_execution',
+    scenarios: ['long_range_trend_analysis', 'topic_evolution', 'multi_condition_filter'],
+    expectedEvidenceCoverage: [
+      'Segment the year into meaningful project phases.',
+      'Search project-related terms and inspect context around disagreements.',
+      'Connect conclusions to cited messages instead of only reporting counts.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-017',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '如果直接搜“团建预算”找不到结果，请换几个相关说法继续查，最后告诉我有没有讨论过。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['search_failure_recovery'],
+    expectedEvidenceCoverage: [
+      'Try the exact query first and record whether it finds evidence.',
+      'Retry with semantically related terms such as activity, reimbursement, cost, or venue.',
+      'Distinguish no evidence found from evidence of absence.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-018',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '筛选去年 9 月到 12 月里，张三参与、和“考试”相关、并且有人回复的问题，总结主要矛盾。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['multi_condition_filter', 'member_lookup_then_messages'],
+    expectedEvidenceCoverage: [
+      'Resolve Zhang San to member identity before filtering messages.',
+      'Apply time, topic, participant, and reply/context constraints together.',
+      'Summarize only conflicts supported by retrieved conversation context.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-019',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '判断大家是不是从 5 月开始明显减少讨论 AI 了；如果证据不够，也请说明不够。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['long_range_trend_analysis', 'insufficient_evidence'],
+    expectedEvidenceCoverage: [
+      'Compare AI-related mentions before and after May with consistent criteria.',
+      'Check enough date coverage to avoid overfitting a short window.',
+      'Explicitly state uncertainty when message volume or query coverage is insufficient.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-020',
+    locale: 'zh-CN',
+    chatType: 'private',
+    userMessage: '总结我和对方关系的变化：从刚开始聊天到最近，有哪些明显转折点？',
+    expectedRoute: 'planned_execution',
+    scenarios: ['relationship_analysis', 'topic_evolution', 'long_range_trend_analysis'],
+    expectedEvidenceCoverage: [
+      'Sample early, middle, and recent conversation periods.',
+      'Identify relationship changes through message context, not only sentiment words.',
+      'Cite concrete exchanges for each proposed turning point.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-021',
+    locale: 'en-US',
+    chatType: 'group',
+    userMessage: 'Compare the top discussion themes in the first half and second half of the archive, with evidence.',
+    expectedRoute: 'planned_execution',
+    scenarios: ['long_range_trend_analysis', 'topic_evolution'],
+    expectedEvidenceCoverage: [
+      'Split the archive into two comparable time windows.',
+      'Identify themes with retrieval or statistics in both windows.',
+      'Provide evidence for differences and avoid unsupported causal claims.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-022',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '这个群里有没有人长期被忽略？请结合发言、回复和互动情况分析。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['key_member_influence', 'relationship_analysis', 'insufficient_evidence'],
+    expectedEvidenceCoverage: [
+      'Use interaction or reply statistics to find low-response candidates.',
+      'Inspect representative contexts before labeling anyone ignored.',
+      'State limitations because absence of replies can be ambiguous.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-023',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '最近有没有提过“签证”？如果没有，帮我查查可能相关的“护照”“出行”“材料”。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['search_failure_recovery', 'simple_search'],
+    expectedEvidenceCoverage: [
+      'Search the exact keyword before related alternatives.',
+      'Inspect related keyword results for actual relevance.',
+      'Report whether evidence is direct, indirect, or absent.',
+    ],
+    baseline: pendingBaseline,
+  },
+  {
+    id: 'ai-route-eval-024',
+    locale: 'zh-CN',
+    chatType: 'group',
+    userMessage: '按月份分析过去一年谁在技术讨论里最活跃，以及他们关注的话题有没有变化。',
+    expectedRoute: 'planned_execution',
+    scenarios: ['long_range_trend_analysis', 'key_member_influence', 'multi_condition_filter'],
+    expectedEvidenceCoverage: [
+      'Filter technical discussions consistently across the full year.',
+      'Compare monthly member activity rather than only global totals.',
+      'Inspect topic evidence for top members across different months.',
+    ],
+    baseline: pendingBaseline,
+  },
+]