feat(tools,server): integrate preprocessing pipeline for web tool results

- Extend ToolResult with optional rawMessages field for structured message data
- Update search and recent-messages tools to return rawMessages from query results
- Add preprocessing wrapper in server tool-adapter: auto-apply data cleaning,
  denoising, desensitization, compact formatting, and token-aware truncation
  when tools return rawMessages
This commit is contained in:
digua committed 2026-05-14 22:04:54 +08:00
1 parent 07056a2258
commit 245a2e39a0
4 files changed
+99

No files matched your search

+69
View File
@@ -2,11 +2,28 @@
* 工具适配层
*
* 将 @openchatlab/tools 的 ToolDefinition 适配为 @mariozechner/pi-agent-core 的 AgentTool 格式。
* 消息类工具返回 rawMessages 时自动执行预处理管道(清洗、去噪、脱敏、截断、格式化)。
*/
import type { AgentTool, AgentToolResult } from '@mariozechner/pi-agent-core'
import type { ToolDefinition, ToolExecutionContext } from '@openchatlab/tools'
import type { DatabaseAdapter } from '@openchatlab/core'
import {
preprocessMessages,
formatMessageCompact,
truncateFormattedMessages,
formatToolResultAsText,
countTokens,
type PreprocessableMessage,
type TruncationStrategy,
} from '@openchatlab/node-runtime'
const DEFAULT_MAX_TOOL_RESULT_TOKENS = 8000
const TOOL_TRUNCATION_STRATEGY: Record<string, TruncationStrategy> = {
chatlab_search: 'keep_first',
chatlab_recent_messages: 'keep_last',
}
export interface ServerToolContext {
db: DatabaseAdapter
@@ -26,6 +43,47 @@ function convertJsonSchemaToParameters(schema: ToolDefinition['inputSchema']) {
}
}
function applyPreprocessing(
rawMessages: PreprocessableMessage[],
toolName: string,
locale?: string,
details?: Record<string, unknown>
): string {
const processed = preprocessMessages(rawMessages)
let formatted = processed.map((m) => formatMessageCompact(m, locale))
let wasTruncated = false
const originalCount = formatted.length
const budget = DEFAULT_MAX_TOOL_RESULT_TOKENS
let totalTokens = 0
for (const line of formatted) {
totalTokens += countTokens(line) + 1
}
if (totalTokens > budget) {
const strategy = TOOL_TRUNCATION_STRATEGY[toolName] ?? 'keep_last'
const truncResult = truncateFormattedMessages(formatted, budget, strategy, countTokens)
if (truncResult.wasTruncated) {
formatted = truncResult.messages
wasTruncated = true
}
}
const finalDetails = { ...details, messages: formatted, returned: formatted.length }
const { rawMessages: _, ...restDetails } = finalDetails as Record<string, unknown>
let textContent = formatToolResultAsText({ ...restDetails, messages: formatted, returned: formatted.length })
if (wasTruncated) {
const strategy = TOOL_TRUNCATION_STRATEGY[toolName] ?? 'keep_last'
const strategyDesc = strategy === 'keep_first' ? 'most relevant' : 'most recent'
const notice = `⚠️ Results truncated: ${originalCount} messages found, showing ${formatted.length} ${strategyDesc} due to context limit. Use a narrower time range or more specific keywords for more precise results.`
textContent = notice + '\n' + textContent
}
return textContent
}
export function adaptToolsForAgent(
tools: ToolDefinition[],
getContext: () => ServerToolContext
@@ -44,6 +102,17 @@ export function adaptToolsForAgent(
}
try {
const result = tool.handler(params, execCtx)
if (result.rawMessages && result.rawMessages.length > 0) {
const textContent = applyPreprocessing(
result.rawMessages as PreprocessableMessage[],
tool.name,
ctx.locale,
(result.data ?? {}) as Record<string, unknown>
)
return { content: [{ type: 'text', text: textContent }], details: null }
}
return { content: [{ type: 'text', text: result.content }], details: null }
} catch (error) {
const msg = error instanceof Error ? error.message : String(error)
@@ -34,6 +34,14 @@ function handler(params: Record<string, unknown>, context: ToolExecutionContext)
return {
content: JSON.stringify(data),
data,
rawMessages: messages.map((m) => ({
id: m.id,
senderId: m.senderId,
senderName: m.senderName,
senderPlatformId: m.senderPlatformId,
content: m.content,
timestamp: m.timestamp,
})),
}
}
+8
View File
@@ -43,6 +43,14 @@ function handler(params: Record<string, unknown>, context: ToolExecutionContext)
return {
content: JSON.stringify(data),
data,
rawMessages: result.messages.map((m) => ({
id: m.id,
senderId: m.senderId,
senderName: m.senderName,
senderPlatformId: m.senderPlatformId,
content: m.content,
timestamp: m.timestamp,
})),
}
}
+14
View File
@@ -35,12 +35,26 @@ export interface ToolExecutionContext {
locale?: string
}
/**
* 可预处理的原始消息(与 @openchatlab/node-runtime PreprocessableMessage 兼容)
*/
export interface RawMessage {
id?: number
senderId?: number
senderName: string
senderPlatformId?: string
content: string | null
timestamp: number
}
/**
* 工具执行结果
*/
export interface ToolResult {
content: string
data?: unknown
/** 消息类工具可透传原始消息数据,供预处理管道消费 */
rawMessages?: RawMessage[]
}
/**