mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
feat(ai): add slash-activated chart runtime with EChart init fix
- Add ChartSpec validation and chart data normalization (bar/line/pie/heatmap) - Add render_chart analysis tool with SQL safety (readonly check, outer LIMIT wrapper, CTE write rejection) - Add chart content block type and ChartBlockRenderer for inline chart rendering - Add slash menu chart capability activation (single-use per message) - Add chart payload transport through Electron and CLI Web tool adapters - Add error block handling for render-only chart tool failures - Add chart payload persistence through content_blocks JSON storage - Fix EChart.vue init timing: defer initChart() to nextTick so container layout completes before ECharts reads width (was rendering 92px canvases) - Add comprehensive test coverage: core, integration, agent flow, frontend, smoke - Add i18n for render_chart and get_schema in 4 locales - Add project handoff documentation
This commit is contained in:
1 parent
dbc377854a
commit
782f67ba63
46 files changed
+2631
-73
No files matched your search
@@ -0,0 +1,381 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import {
|
||||
createAssistantMessageEventStream,
|
||||
type AssistantMessage,
|
||||
type Context,
|
||||
type Message,
|
||||
type Model,
|
||||
type ToolCall,
|
||||
type Usage,
|
||||
} from '@earendil-works/pi-ai'
|
||||
import type { AgentTool, AgentToolResult } from '@earendil-works/pi-agent-core'
|
||||
import {
|
||||
renderChartTool,
|
||||
type ToolDataProvider,
|
||||
type ToolDefinition,
|
||||
type ToolExecutionContext,
|
||||
} from '@openchatlab/tools'
|
||||
import type { ChartPayload } from '@openchatlab/core'
|
||||
import { runAgentCore, type AgentCoreEvent } from '../../packages/node-runtime/src/ai/agent'
|
||||
|
||||
function emptyUsage(): Usage {
|
||||
return {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
}
|
||||
}
|
||||
|
||||
function assistantMessage(
|
||||
content: AssistantMessage['content'],
|
||||
stopReason: AssistantMessage['stopReason']
|
||||
): AssistantMessage {
|
||||
return {
|
||||
role: 'assistant',
|
||||
content,
|
||||
api: 'openai-completions',
|
||||
provider: 'chatlab-test',
|
||||
model: 'fake-chart-model',
|
||||
usage: emptyUsage(),
|
||||
stopReason,
|
||||
timestamp: Date.now(),
|
||||
}
|
||||
}
|
||||
|
||||
function createToolCallsStream(toolCalls: ToolCall[]) {
|
||||
const stream = createAssistantMessageEventStream()
|
||||
const partialStart = assistantMessage([], 'toolUse')
|
||||
const finalMessage = assistantMessage(toolCalls, 'toolUse')
|
||||
stream.push({ type: 'start', partial: partialStart })
|
||||
toolCalls.forEach((toolCall, contentIndex) => {
|
||||
stream.push({ type: 'toolcall_start', contentIndex, partial: finalMessage })
|
||||
stream.push({
|
||||
type: 'toolcall_delta',
|
||||
contentIndex,
|
||||
delta: JSON.stringify(toolCall.arguments),
|
||||
partial: finalMessage,
|
||||
})
|
||||
stream.push({ type: 'toolcall_end', contentIndex, toolCall, partial: finalMessage })
|
||||
})
|
||||
stream.push({ type: 'done', reason: 'toolUse', message: finalMessage })
|
||||
return stream
|
||||
}
|
||||
|
||||
function createToolCallStream(toolCall: ToolCall) {
|
||||
return createToolCallsStream([toolCall])
|
||||
}
|
||||
|
||||
function createTextStream(text: string) {
|
||||
const stream = createAssistantMessageEventStream()
|
||||
const partialStart = assistantMessage([], 'stop')
|
||||
const finalMessage = assistantMessage([{ type: 'text', text }], 'stop')
|
||||
stream.push({ type: 'start', partial: partialStart })
|
||||
stream.push({ type: 'text_start', contentIndex: 0, partial: partialStart })
|
||||
stream.push({ type: 'text_delta', contentIndex: 0, delta: text, partial: finalMessage })
|
||||
stream.push({ type: 'text_end', contentIndex: 0, content: text, partial: finalMessage })
|
||||
stream.push({ type: 'done', reason: 'stop', message: finalMessage })
|
||||
return stream
|
||||
}
|
||||
|
||||
function hasToolResult(messages: Message[]): boolean {
|
||||
return messages.some((message) => message.role === 'toolResult' && message.toolName === 'render_chart')
|
||||
}
|
||||
|
||||
function countToolResults(messages: Message[]): number {
|
||||
return messages.filter((message) => message.role === 'toolResult' && message.toolName === 'render_chart').length
|
||||
}
|
||||
|
||||
function convertJsonSchemaToParameters(schema: ToolDefinition['inputSchema']) {
|
||||
const properties: Record<string, unknown> = {}
|
||||
for (const [key, prop] of Object.entries(schema.properties)) {
|
||||
properties[key] = { ...prop }
|
||||
}
|
||||
return {
|
||||
type: 'object' as const,
|
||||
properties,
|
||||
required: schema.required || [],
|
||||
}
|
||||
}
|
||||
|
||||
function adaptRenderChartTool(dataProvider: ToolDataProvider): AgentTool<any, unknown> {
|
||||
return {
|
||||
name: renderChartTool.name,
|
||||
label: renderChartTool.name,
|
||||
description: renderChartTool.description,
|
||||
parameters: convertJsonSchemaToParameters(renderChartTool.inputSchema) as any,
|
||||
async execute(_toolCallId: string, params: unknown): Promise<AgentToolResult<unknown>> {
|
||||
const context: ToolExecutionContext = {
|
||||
sessionId: 'offline-agent-chart-flow',
|
||||
locale: 'en-US',
|
||||
dataProvider,
|
||||
}
|
||||
const result = await renderChartTool.handler((params ?? {}) as Record<string, unknown>, context)
|
||||
return {
|
||||
content: [{ type: 'text', text: result.content }],
|
||||
details: {
|
||||
...(typeof result.data === 'object' && result.data !== null ? result.data : {}),
|
||||
...(result.chart ? { chart: result.chart } : {}),
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe('offline Agent chart flow', () => {
|
||||
it('turns a natural-language chart request into a render_chart tool result and final answer', async () => {
|
||||
const requestedSql: string[] = []
|
||||
const dataProvider: ToolDataProvider = {
|
||||
async executeParameterizedSql(sql, params) {
|
||||
requestedSql.push(sql)
|
||||
assert.match(sql, /JOIN member/i)
|
||||
assert.match(sql, /@memberA/)
|
||||
assert.deepEqual(params, { memberA: 'Alice', memberB: 'Bob', startTs: 1780272000, endTs: 1780444800 })
|
||||
return [
|
||||
{ day: '2026-06-01', member_name: 'Alice', message_count: 2 },
|
||||
{ day: '2026-06-01', member_name: 'Bob', message_count: 1 },
|
||||
{ day: '2026-06-02', member_name: 'Alice', message_count: 2 },
|
||||
{ day: '2026-06-02', member_name: 'Bob', message_count: 2 },
|
||||
]
|
||||
},
|
||||
async getSchema() {
|
||||
return []
|
||||
},
|
||||
} as Partial<ToolDataProvider> as ToolDataProvider
|
||||
const tools = [adaptRenderChartTool(dataProvider)]
|
||||
const streamFn = (_model: Model<any>, context: Context) => {
|
||||
const userMessages = context.messages.filter((message) => message.role === 'user')
|
||||
const latestUser = userMessages[userMessages.length - 1]
|
||||
const userText = Array.isArray(latestUser?.content)
|
||||
? latestUser.content.map((part) => (part.type === 'text' ? part.text : '')).join('\n')
|
||||
: latestUser?.content || ''
|
||||
assert.match(userText, /Alice/)
|
||||
assert.match(userText, /Bob/)
|
||||
assert.match(userText, /line/i)
|
||||
|
||||
if (!hasToolResult(context.messages)) {
|
||||
return createToolCallStream({
|
||||
type: 'toolCall',
|
||||
id: 'call_chart_1',
|
||||
name: 'render_chart',
|
||||
arguments: {
|
||||
sql: `
|
||||
SELECT date(msg.ts, 'unixepoch') AS day,
|
||||
COALESCE(m.group_nickname, m.account_name, m.platform_id) AS member_name,
|
||||
COUNT(*) AS message_count
|
||||
FROM message msg
|
||||
JOIN member m ON m.id = msg.sender_id
|
||||
WHERE COALESCE(m.group_nickname, m.account_name, m.platform_id) IN (@memberA, @memberB)
|
||||
AND msg.ts >= @startTs AND msg.ts < @endTs
|
||||
GROUP BY day, m.id
|
||||
ORDER BY day ASC, member_name ASC
|
||||
`,
|
||||
params: { memberA: 'Alice', memberB: 'Bob', startTs: 1780272000, endTs: 1780444800 },
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'line',
|
||||
title: 'Alice vs Bob daily message trend',
|
||||
encoding: { x: 'day', y: 'message_count', series: 'member_name' },
|
||||
filters: { members: [{ name: 'Alice' }, { name: 'Bob' }] },
|
||||
unit: 'messages',
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return createTextStream('Alice stayed ahead on both days, while Bob caught up on June 2.')
|
||||
}
|
||||
const events: AgentCoreEvent[] = []
|
||||
|
||||
const result = await runAgentCore({
|
||||
piModel: {
|
||||
id: 'fake-chart-model',
|
||||
provider: 'chatlab-test',
|
||||
api: 'openai-completions',
|
||||
} as Model<any>,
|
||||
apiKey: 'test-key',
|
||||
systemPrompt: 'Use render_chart for chart requests.',
|
||||
tools,
|
||||
history: [],
|
||||
userMessage: 'Please draw a line chart comparing Alice and Bob message counts by day.',
|
||||
maxToolRounds: 3,
|
||||
steerMessage: 'Please explain the chart briefly.',
|
||||
streamFn,
|
||||
onEvent: (event) => events.push(event),
|
||||
})
|
||||
|
||||
assert.equal(result.error, undefined)
|
||||
assert.deepEqual(result.toolsUsed, ['render_chart'])
|
||||
assert.equal(result.toolRounds, 1)
|
||||
assert.equal(requestedSql.length, 1)
|
||||
|
||||
const toolEnd = events.find((event) => event.type === 'tool_end' && event.toolName === 'render_chart')
|
||||
assert.ok(toolEnd && 'toolResult' in toolEnd, 'expected render_chart tool_end event')
|
||||
const chart = (toolEnd.toolResult as { details?: { chart?: ChartPayload } }).details?.chart
|
||||
assert.ok(chart, 'expected chart payload in render_chart tool result details')
|
||||
assert.equal(chart.spec.type, 'line')
|
||||
assert.deepEqual(chart.data, {
|
||||
labels: ['2026-06-01', '2026-06-02'],
|
||||
values: [2, 2],
|
||||
series: [
|
||||
{ name: 'Alice', values: [2, 2] },
|
||||
{ name: 'Bob', values: [1, 2] },
|
||||
],
|
||||
})
|
||||
|
||||
const content = events
|
||||
.filter((event) => event.type === 'content')
|
||||
.map((event) => event.content)
|
||||
.join('')
|
||||
assert.match(content, /Bob caught up/)
|
||||
})
|
||||
|
||||
it('runs multiple render_chart calls when the user asks for two charts', async () => {
|
||||
const requestedSql: string[] = []
|
||||
const dataProvider: ToolDataProvider = {
|
||||
async executeParameterizedSql(sql, params) {
|
||||
requestedSql.push(sql)
|
||||
if (sql.includes('selected_member_share_source')) {
|
||||
assert.deepEqual(params, { memberA: 'Alice', memberB: 'Bob' })
|
||||
return [
|
||||
{ member_name: 'Alice', message_count: 4 },
|
||||
{ member_name: 'Bob', message_count: 3 },
|
||||
]
|
||||
}
|
||||
if (sql.includes('member_hour_density_source')) {
|
||||
assert.deepEqual(params, { memberA: 'Alice', memberB: 'Bob' })
|
||||
return [
|
||||
{ hour: '09', member_name: 'Alice', message_count: 2 },
|
||||
{ hour: '10', member_name: 'Alice', message_count: 2 },
|
||||
{ hour: '09', member_name: 'Bob', message_count: 1 },
|
||||
{ hour: '10', member_name: 'Bob', message_count: 2 },
|
||||
]
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`)
|
||||
},
|
||||
async getSchema() {
|
||||
return []
|
||||
},
|
||||
} as Partial<ToolDataProvider> as ToolDataProvider
|
||||
const tools = [adaptRenderChartTool(dataProvider)]
|
||||
const streamFn = (_model: Model<any>, context: Context) => {
|
||||
const userMessages = context.messages.filter((message) => message.role === 'user')
|
||||
const latestUser = userMessages[userMessages.length - 1]
|
||||
const userText = Array.isArray(latestUser?.content)
|
||||
? latestUser.content.map((part) => (part.type === 'text' ? part.text : '')).join('\n')
|
||||
: latestUser?.content || ''
|
||||
assert.match(userText, /two charts/i)
|
||||
assert.match(userText, /pie/i)
|
||||
assert.match(userText, /heatmap/i)
|
||||
|
||||
if (countToolResults(context.messages) < 2) {
|
||||
return createToolCallsStream([
|
||||
{
|
||||
type: 'toolCall',
|
||||
id: 'call_chart_pie',
|
||||
name: 'render_chart',
|
||||
arguments: {
|
||||
sql: `
|
||||
WITH selected_member_share_source AS (
|
||||
SELECT @memberA AS member_name, 4 AS message_count
|
||||
UNION ALL
|
||||
SELECT @memberB AS member_name, 3 AS message_count
|
||||
)
|
||||
SELECT member_name, message_count
|
||||
FROM selected_member_share_source
|
||||
`,
|
||||
params: { memberA: 'Alice', memberB: 'Bob' },
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'pie',
|
||||
title: 'Selected members message share',
|
||||
encoding: { label: 'member_name', value: 'message_count' },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'toolCall',
|
||||
id: 'call_chart_heatmap',
|
||||
name: 'render_chart',
|
||||
arguments: {
|
||||
sql: `
|
||||
WITH member_hour_density_source AS (
|
||||
SELECT '09' AS hour, @memberA AS member_name, 2 AS message_count
|
||||
UNION ALL
|
||||
SELECT '10' AS hour, @memberA AS member_name, 2 AS message_count
|
||||
UNION ALL
|
||||
SELECT '09' AS hour, @memberB AS member_name, 1 AS message_count
|
||||
UNION ALL
|
||||
SELECT '10' AS hour, @memberB AS member_name, 2 AS message_count
|
||||
)
|
||||
SELECT hour, member_name, message_count
|
||||
FROM member_hour_density_source
|
||||
`,
|
||||
params: { memberA: 'Alice', memberB: 'Bob' },
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'heatmap',
|
||||
title: 'Selected members hour density',
|
||||
encoding: { x: 'hour', y: 'member_name', value: 'message_count' },
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
return createTextStream('Here are the requested pie chart and heatmap.')
|
||||
}
|
||||
const events: AgentCoreEvent[] = []
|
||||
|
||||
const result = await runAgentCore({
|
||||
piModel: {
|
||||
id: 'fake-chart-model',
|
||||
provider: 'chatlab-test',
|
||||
api: 'openai-completions',
|
||||
} as Model<any>,
|
||||
apiKey: 'test-key',
|
||||
systemPrompt: 'Use render_chart for chart requests.',
|
||||
tools,
|
||||
history: [],
|
||||
userMessage: 'Please generate two charts: a pie chart for Alice and Bob, plus a heatmap by hour.',
|
||||
maxToolRounds: 3,
|
||||
steerMessage: 'Please explain both charts briefly.',
|
||||
streamFn,
|
||||
onEvent: (event) => events.push(event),
|
||||
})
|
||||
|
||||
assert.equal(result.error, undefined)
|
||||
assert.deepEqual(result.toolsUsed, ['render_chart', 'render_chart'])
|
||||
assert.equal(result.toolRounds, 1)
|
||||
assert.equal(requestedSql.length, 2)
|
||||
|
||||
const chartPayloads = events
|
||||
.filter((event) => event.type === 'tool_end' && event.toolName === 'render_chart')
|
||||
.map((event) => (event.toolResult as { details?: { chart?: ChartPayload } }).details?.chart)
|
||||
.filter((chart): chart is ChartPayload => !!chart)
|
||||
|
||||
assert.equal(chartPayloads.length, 2)
|
||||
const chartsByType = new Map(chartPayloads.map((chart) => [chart.spec.type, chart]))
|
||||
assert.deepEqual([...chartsByType.keys()].sort(), ['heatmap', 'pie'])
|
||||
assert.deepEqual(chartsByType.get('pie')?.data, { labels: ['Alice', 'Bob'], values: [4, 3] })
|
||||
assert.deepEqual(chartsByType.get('heatmap')?.data, {
|
||||
xLabels: ['09', '10'],
|
||||
yLabels: ['Alice', 'Bob'],
|
||||
data: [
|
||||
[0, 0, 2],
|
||||
[1, 0, 2],
|
||||
[0, 1, 1],
|
||||
[1, 1, 2],
|
||||
],
|
||||
})
|
||||
|
||||
const content = events
|
||||
.filter((event) => event.type === 'content')
|
||||
.map((event) => event.content)
|
||||
.join('')
|
||||
assert.match(content, /pie chart and heatmap/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,198 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { existsSync, mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { AgentTool, AgentToolResult } from '@earendil-works/pi-agent-core'
|
||||
import { CHAT_DB_SCHEMA, getChartCapabilitySkill, type ChartPayload } from '@openchatlab/core'
|
||||
import {
|
||||
CoreDataProvider,
|
||||
renderChartTool,
|
||||
schemaTool,
|
||||
type ToolDataProvider,
|
||||
type ToolDefinition,
|
||||
type ToolExecutionContext,
|
||||
} from '@openchatlab/tools'
|
||||
import { buildPiModel } from '../../packages/node-runtime/src/ai/llm-builder'
|
||||
import { runAgentCore, type AgentCoreEvent } from '../../packages/node-runtime/src/ai/agent'
|
||||
import { openBetterSqliteDatabase } from '../../packages/node-runtime/src/better-sqlite3-adapter'
|
||||
|
||||
const shouldRunRealLlmE2E = process.env.CHATLAB_RUN_REAL_LLM_CHART_E2E === '1'
|
||||
|
||||
function resolveNativeBinding(): string | undefined {
|
||||
if (process.env.CHATLAB_TEST_SQLITE_NATIVE_BINDING) return process.env.CHATLAB_TEST_SQLITE_NATIVE_BINDING
|
||||
const repoNativeBinding = path.resolve('apps/cli/native/better_sqlite3.node')
|
||||
return existsSync(repoNativeBinding) ? repoNativeBinding : undefined
|
||||
}
|
||||
|
||||
function unixTs(iso: string): number {
|
||||
return Math.floor(Date.parse(iso) / 1000)
|
||||
}
|
||||
|
||||
function createTempDir(): string {
|
||||
return mkdtempSync(path.join(tmpdir(), 'chatlab-real-llm-chart-e2e-'))
|
||||
}
|
||||
|
||||
function seedChatDb(db: ReturnType<typeof openBetterSqliteDatabase>): void {
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
db.prepare(
|
||||
`INSERT INTO meta (name, platform, type, imported_at, group_id, owner_id)
|
||||
VALUES ('Chart E2E Group', 'wechat', 'group', ?, 'g1', 'u_alice')`
|
||||
).run(unixTs('2026-06-03T00:00:00Z'))
|
||||
|
||||
const insertMember = db.prepare(
|
||||
`INSERT INTO member (platform_id, account_name, group_nickname)
|
||||
VALUES (?, ?, ?)`
|
||||
)
|
||||
insertMember.run('u_alice', 'Alice Account', 'Alice')
|
||||
insertMember.run('u_bob', 'Bob Account', 'Bob')
|
||||
insertMember.run('u_cara', 'Cara Account', 'Cara')
|
||||
|
||||
const memberRows = db.prepare('SELECT id, group_nickname FROM member').all() as Array<{
|
||||
id: number
|
||||
group_nickname: string
|
||||
}>
|
||||
const memberId = new Map(memberRows.map((row) => [row.group_nickname, row.id]))
|
||||
const insertMessage = db.prepare(
|
||||
`INSERT INTO message (sender_id, sender_account_name, sender_group_nickname, ts, type, content, platform_message_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
const add = (name: 'Alice' | 'Bob' | 'Cara', ts: string, type: number, content: string) => {
|
||||
const id = memberId.get(name)
|
||||
assert.ok(id, `missing member id for ${name}`)
|
||||
insertMessage.run(id, `${name} Account`, name, unixTs(ts), type, content, `${name}-${ts}`)
|
||||
}
|
||||
|
||||
add('Alice', '2026-06-01T09:00:00Z', 0, 'morning hello')
|
||||
add('Alice', '2026-06-01T10:00:00Z', 0, 'standup note')
|
||||
add('Alice', '2026-06-02T09:00:00Z', 1, '[图片]')
|
||||
add('Alice', '2026-06-02T10:00:00Z', 0, 'follow up')
|
||||
add('Bob', '2026-06-01T09:00:00Z', 0, 'first reply')
|
||||
add('Bob', '2026-06-02T10:00:00Z', 0, 'second reply')
|
||||
add('Bob', '2026-06-02T11:00:00Z', 0, 'third reply')
|
||||
add('Cara', '2026-06-01T09:00:00Z', 0, 'other member')
|
||||
add('Cara', '2026-06-03T12:00:00Z', 0, 'outside selected members')
|
||||
}
|
||||
|
||||
function convertJsonSchemaToParameters(schema: ToolDefinition['inputSchema']) {
|
||||
const properties: Record<string, unknown> = {}
|
||||
for (const [key, prop] of Object.entries(schema.properties)) {
|
||||
properties[key] = { ...prop }
|
||||
}
|
||||
return {
|
||||
type: 'object' as const,
|
||||
properties,
|
||||
required: schema.required || [],
|
||||
}
|
||||
}
|
||||
|
||||
function adaptTool(tool: ToolDefinition, dataProvider: ToolDataProvider): AgentTool<any, unknown> {
|
||||
return {
|
||||
name: tool.name,
|
||||
label: tool.name,
|
||||
description: tool.description,
|
||||
parameters: convertJsonSchemaToParameters(tool.inputSchema) as any,
|
||||
async execute(_toolCallId: string, params: unknown): Promise<AgentToolResult<unknown>> {
|
||||
const context: ToolExecutionContext = {
|
||||
sessionId: 'real-llm-chart-flow',
|
||||
locale: 'en-US',
|
||||
dataProvider,
|
||||
}
|
||||
const result = await tool.handler((params ?? {}) as Record<string, unknown>, context)
|
||||
return {
|
||||
content: [{ type: 'text', text: result.content }],
|
||||
details: {
|
||||
...(typeof result.data === 'object' && result.data !== null && !Array.isArray(result.data)
|
||||
? result.data
|
||||
: {}),
|
||||
...(result.chart ? { chart: result.chart } : {}),
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function getChartPayloads(events: AgentCoreEvent[]): ChartPayload[] {
|
||||
return events
|
||||
.filter((event) => event.type === 'tool_end' && event.toolName === 'render_chart')
|
||||
.map((event) => (event.toolResult as { details?: { chart?: ChartPayload } }).details?.chart)
|
||||
.filter((chart): chart is ChartPayload => !!chart)
|
||||
}
|
||||
|
||||
describe('real external LLM chart flow', () => {
|
||||
it(
|
||||
'asks a real model to generate two chart payloads from real chat data',
|
||||
{ skip: !shouldRunRealLlmE2E },
|
||||
async () => {
|
||||
const apiKey = process.env.CHATLAB_REAL_LLM_API_KEY || process.env.OPENAI_API_KEY
|
||||
assert.ok(apiKey, 'Set CHATLAB_REAL_LLM_API_KEY or OPENAI_API_KEY to run the real LLM chart E2E test')
|
||||
|
||||
const provider = process.env.CHATLAB_REAL_LLM_PROVIDER || 'openai'
|
||||
const model = process.env.CHATLAB_REAL_LLM_MODEL || 'gpt-4.1-mini'
|
||||
const baseUrl = process.env.CHATLAB_REAL_LLM_BASE_URL || undefined
|
||||
const piModel = buildPiModel({ provider, model, baseUrl })
|
||||
const dir = createTempDir()
|
||||
const dbPath = path.join(dir, 'chat.db')
|
||||
const nativeBinding = resolveNativeBinding()
|
||||
const db = openBetterSqliteDatabase(dbPath, { nativeBinding })
|
||||
|
||||
try {
|
||||
seedChatDb(db)
|
||||
const dataProvider = new CoreDataProvider(db)
|
||||
const tools = [adaptTool(schemaTool, dataProvider), adaptTool(renderChartTool, dataProvider)]
|
||||
const events: AgentCoreEvent[] = []
|
||||
const chartPrompt = getChartCapabilitySkill('en-US').prompt
|
||||
|
||||
const result = await runAgentCore({
|
||||
piModel,
|
||||
apiKey,
|
||||
systemPrompt: `${chartPrompt}
|
||||
|
||||
Test-specific requirements:
|
||||
- Use get_schema if you need table names or columns.
|
||||
- Produce exactly two render_chart calls.
|
||||
- The pie chart must include only Alice and Bob and must exclude Cara.
|
||||
- The heatmap must include only Alice and Bob and use hour on x, member name on y, and message count as value.
|
||||
- Use the data from June 1-2, 2026 inclusive. The database stores Unix seconds in message.ts.`,
|
||||
tools,
|
||||
history: [],
|
||||
userMessage:
|
||||
'Create exactly two charts from this chat data: first a pie chart comparing Alice and Bob message share, then a heatmap showing Alice and Bob message density by hour. Briefly explain both.',
|
||||
maxToolRounds: 4,
|
||||
steerMessage: 'Explain the generated charts briefly. Do not call more tools.',
|
||||
thinkingLevel: 'off',
|
||||
onEvent: (event) => events.push(event),
|
||||
})
|
||||
|
||||
assert.equal(result.error, undefined)
|
||||
const charts = getChartPayloads(events)
|
||||
assert.equal(charts.length, 2, `expected exactly two chart payloads, got ${charts.length}`)
|
||||
|
||||
const chartsByType = new Map(charts.map((chart) => [chart.spec.type, chart]))
|
||||
const pie = chartsByType.get('pie')
|
||||
const heatmap = chartsByType.get('heatmap')
|
||||
assert.ok(pie, 'expected a pie chart payload')
|
||||
assert.ok(heatmap, 'expected a heatmap chart payload')
|
||||
assert.deepEqual(new Set(pie.data.labels), new Set(['Alice', 'Bob']))
|
||||
assert.equal(
|
||||
pie.data.values.reduce((sum, value) => sum + value, 0),
|
||||
7
|
||||
)
|
||||
assert.ok(!heatmap.data.yLabels.includes('Cara'), 'heatmap should exclude Cara')
|
||||
assert.ok(heatmap.data.yLabels.includes('Alice'), 'heatmap should include Alice')
|
||||
assert.ok(heatmap.data.yLabels.includes('Bob'), 'heatmap should include Bob')
|
||||
assert.ok(heatmap.data.xLabels.includes('09'), 'heatmap should include hour 09')
|
||||
assert.ok(heatmap.data.data.length > 0, 'heatmap should include density cells')
|
||||
|
||||
const finalContent = events
|
||||
.filter((event) => event.type === 'content')
|
||||
.map((event) => event.content)
|
||||
.join('')
|
||||
assert.match(finalContent, /Alice|Bob|pie|heatmap/i)
|
||||
} finally {
|
||||
db.close()
|
||||
rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 })
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,243 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { existsSync, mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { CHAT_DB_SCHEMA, type ChartPayload } from '@openchatlab/core'
|
||||
import { CoreDataProvider, renderChartTool } from '@openchatlab/tools'
|
||||
import { openBetterSqliteDatabase } from '../../packages/node-runtime/src/better-sqlite3-adapter'
|
||||
|
||||
function resolveNativeBinding(): string | undefined {
|
||||
if (process.env.CHATLAB_TEST_SQLITE_NATIVE_BINDING) return process.env.CHATLAB_TEST_SQLITE_NATIVE_BINDING
|
||||
const repoNativeBinding = path.resolve('apps/cli/native/better_sqlite3.node')
|
||||
return existsSync(repoNativeBinding) ? repoNativeBinding : undefined
|
||||
}
|
||||
|
||||
function unixTs(iso: string): number {
|
||||
return Math.floor(Date.parse(iso) / 1000)
|
||||
}
|
||||
|
||||
function createTempDir(): string {
|
||||
return mkdtempSync(path.join(tmpdir(), 'chatlab-chart-runtime-'))
|
||||
}
|
||||
|
||||
function getChart(result: Awaited<ReturnType<typeof renderChartTool.handler>>): ChartPayload {
|
||||
assert.ok(result.chart, 'expected render_chart to return a chart payload')
|
||||
return result.chart
|
||||
}
|
||||
|
||||
function seedChatDb(db: ReturnType<typeof openBetterSqliteDatabase>): void {
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
db.prepare(
|
||||
`INSERT INTO meta (name, platform, type, imported_at, group_id, owner_id)
|
||||
VALUES ('Chart Test Group', 'wechat', 'group', ?, 'g1', 'u_alice')`
|
||||
).run(unixTs('2026-06-03T00:00:00Z'))
|
||||
|
||||
const insertMember = db.prepare(
|
||||
`INSERT INTO member (platform_id, account_name, group_nickname)
|
||||
VALUES (?, ?, ?)`
|
||||
)
|
||||
insertMember.run('u_alice', 'Alice Account', 'Alice')
|
||||
insertMember.run('u_bob', 'Bob Account', 'Bob')
|
||||
insertMember.run('u_cara', 'Cara Account', 'Cara')
|
||||
|
||||
const memberRows = db.prepare('SELECT id, group_nickname FROM member').all() as Array<{
|
||||
id: number
|
||||
group_nickname: string
|
||||
}>
|
||||
const memberId = new Map(memberRows.map((row) => [row.group_nickname, row.id]))
|
||||
const insertMessage = db.prepare(
|
||||
`INSERT INTO message (sender_id, sender_account_name, sender_group_nickname, ts, type, content, platform_message_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
const add = (name: 'Alice' | 'Bob' | 'Cara', ts: string, type: number, content: string) => {
|
||||
const id = memberId.get(name)
|
||||
assert.ok(id, `missing member id for ${name}`)
|
||||
insertMessage.run(id, `${name} Account`, name, unixTs(ts), type, content, `${name}-${ts}`)
|
||||
}
|
||||
|
||||
add('Alice', '2026-06-01T09:00:00Z', 0, 'morning hello')
|
||||
add('Alice', '2026-06-01T10:00:00Z', 0, 'standup note')
|
||||
add('Alice', '2026-06-02T09:00:00Z', 1, '[图片]')
|
||||
add('Alice', '2026-06-02T10:00:00Z', 0, 'follow up')
|
||||
add('Bob', '2026-06-01T09:00:00Z', 0, 'first reply')
|
||||
add('Bob', '2026-06-02T10:00:00Z', 0, 'second reply')
|
||||
add('Bob', '2026-06-02T11:00:00Z', 0, 'third reply')
|
||||
add('Cara', '2026-06-01T09:00:00Z', 0, 'other member')
|
||||
add('Cara', '2026-06-03T12:00:00Z', 0, 'outside selected members')
|
||||
}
|
||||
|
||||
describe('render_chart integration with a real chat SQLite database', () => {
|
||||
it('executes read-only SQL against chat tables and returns flexible chart payloads', async () => {
|
||||
const dir = createTempDir()
|
||||
const dbPath = path.join(dir, 'chat.db')
|
||||
const nativeBinding = resolveNativeBinding()
|
||||
const db = openBetterSqliteDatabase(dbPath, { nativeBinding })
|
||||
|
||||
try {
|
||||
seedChatDb(db)
|
||||
const dataProvider = new CoreDataProvider(db)
|
||||
const context = {
|
||||
sessionId: 'chart-test-session',
|
||||
locale: 'en-US',
|
||||
dataProvider,
|
||||
}
|
||||
const range = {
|
||||
startTs: unixTs('2026-06-01T00:00:00Z'),
|
||||
endTs: unixTs('2026-06-03T00:00:00Z'),
|
||||
}
|
||||
|
||||
const pie = getChart(
|
||||
await renderChartTool.handler(
|
||||
{
|
||||
sql: `
|
||||
SELECT COALESCE(m.group_nickname, m.account_name, m.platform_id) AS name,
|
||||
COUNT(*) AS message_count
|
||||
FROM message msg
|
||||
JOIN member m ON m.id = msg.sender_id
|
||||
WHERE COALESCE(m.group_nickname, m.account_name, m.platform_id) IN (@memberA, @memberB)
|
||||
AND msg.ts >= @startTs AND msg.ts < @endTs
|
||||
GROUP BY m.id
|
||||
ORDER BY message_count DESC, name ASC
|
||||
`,
|
||||
params: { memberA: 'Alice', memberB: 'Bob', ...range },
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'pie',
|
||||
title: 'Selected members message share',
|
||||
encoding: { label: 'name', value: 'message_count' },
|
||||
filters: { members: [{ name: 'Alice' }, { name: 'Bob' }] },
|
||||
},
|
||||
},
|
||||
context
|
||||
)
|
||||
)
|
||||
assert.equal(pie.spec.type, 'pie')
|
||||
assert.deepEqual(pie.data, { labels: ['Alice', 'Bob'], values: [4, 3] })
|
||||
assert.deepEqual(
|
||||
pie.spec.filters?.members?.map((member) => member.name),
|
||||
['Alice', 'Bob']
|
||||
)
|
||||
|
||||
const line = getChart(
|
||||
await renderChartTool.handler(
|
||||
{
|
||||
sql: `
|
||||
SELECT date(msg.ts, 'unixepoch') AS day,
|
||||
COALESCE(m.group_nickname, m.account_name, m.platform_id) AS member_name,
|
||||
COUNT(*) AS message_count
|
||||
FROM message msg
|
||||
JOIN member m ON m.id = msg.sender_id
|
||||
WHERE COALESCE(m.group_nickname, m.account_name, m.platform_id) IN (@memberA, @memberB)
|
||||
AND msg.ts >= @startTs AND msg.ts < @endTs
|
||||
GROUP BY day, m.id
|
||||
ORDER BY day ASC, member_name ASC
|
||||
`,
|
||||
params: { memberA: 'Alice', memberB: 'Bob', ...range },
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'line',
|
||||
title: 'Selected members daily trend',
|
||||
encoding: { x: 'day', y: 'message_count', series: 'member_name' },
|
||||
},
|
||||
},
|
||||
context
|
||||
)
|
||||
)
|
||||
assert.equal(line.spec.type, 'line')
|
||||
assert.deepEqual(line.data, {
|
||||
labels: ['2026-06-01', '2026-06-02'],
|
||||
values: [2, 2],
|
||||
series: [
|
||||
{ name: 'Alice', values: [2, 2] },
|
||||
{ name: 'Bob', values: [1, 2] },
|
||||
],
|
||||
})
|
||||
|
||||
const heatmap = getChart(
|
||||
await renderChartTool.handler(
|
||||
{
|
||||
sql: `
|
||||
SELECT strftime('%H', msg.ts, 'unixepoch') AS hour,
|
||||
COALESCE(m.group_nickname, m.account_name, m.platform_id) AS member_name,
|
||||
COUNT(*) AS message_count
|
||||
FROM message msg
|
||||
JOIN member m ON m.id = msg.sender_id
|
||||
WHERE msg.ts >= @startTs AND msg.ts < @endTs
|
||||
GROUP BY hour, m.id
|
||||
ORDER BY member_name ASC, hour ASC
|
||||
`,
|
||||
params: range,
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'heatmap',
|
||||
title: 'Member hour density',
|
||||
encoding: { x: 'hour', y: 'member_name', value: 'message_count' },
|
||||
},
|
||||
},
|
||||
context
|
||||
)
|
||||
)
|
||||
assert.equal(heatmap.spec.type, 'heatmap')
|
||||
assert.deepEqual(heatmap.data, {
|
||||
xLabels: ['09', '10', '11'],
|
||||
yLabels: ['Alice', 'Bob', 'Cara'],
|
||||
data: [
|
||||
[0, 0, 2],
|
||||
[1, 0, 2],
|
||||
[0, 1, 1],
|
||||
[1, 1, 1],
|
||||
[2, 1, 1],
|
||||
[0, 2, 1],
|
||||
],
|
||||
})
|
||||
} finally {
|
||||
db.close()
|
||||
rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 })
|
||||
}
|
||||
})
|
||||
|
||||
it('does not mutate chat data when a WITH-prefixed write is attempted', async () => {
|
||||
const dir = createTempDir()
|
||||
const dbPath = path.join(dir, 'chat.db')
|
||||
const nativeBinding = resolveNativeBinding()
|
||||
const db = openBetterSqliteDatabase(dbPath, { nativeBinding })
|
||||
|
||||
try {
|
||||
seedChatDb(db)
|
||||
const dataProvider = new CoreDataProvider(db)
|
||||
const context = {
|
||||
sessionId: 'chart-test-session',
|
||||
locale: 'en-US',
|
||||
dataProvider,
|
||||
}
|
||||
const beforeCount = (db.prepare('SELECT COUNT(*) AS count FROM message').get() as { count: number }).count
|
||||
|
||||
await assert.rejects(
|
||||
async () =>
|
||||
renderChartTool.handler(
|
||||
{
|
||||
sql: `
|
||||
WITH victim AS (SELECT id FROM message LIMIT 1)
|
||||
DELETE FROM message WHERE id IN (SELECT id FROM victim)
|
||||
`,
|
||||
chartSpec: {
|
||||
version: 1,
|
||||
type: 'bar',
|
||||
title: 'Should not render',
|
||||
encoding: { x: 'name', y: 'message_count' },
|
||||
},
|
||||
},
|
||||
context
|
||||
),
|
||||
/READ-ONLY|read-only|not authorized|readonly|SELECT|syntax/i
|
||||
)
|
||||
|
||||
const afterCount = (db.prepare('SELECT COUNT(*) AS count FROM message').get() as { count: number }).count
|
||||
assert.equal(afterCount, beforeCount)
|
||||
} finally {
|
||||
db.close()
|
||||
rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 })
|
||||
}
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user