refactor: extract shared preprocessing pipeline to @openchatlab/node-runtime

Unify the duplicated preprocessing logic between Electron (wrapWithPreprocessing)
and Server (applyPreprocessing) into a single applyPreprocessingPipeline() function
in @openchatlab/node-runtime. Both platforms now delegate to the same implementation,
eliminating ~50 lines of redundant code and fixing a redundant token counting bug
in the Server version.

electron/main/ai/tools/index.ts

@@ -11,14 +11,7 @@ import { TOOL_REGISTRY } from './definitions'
 
 const CORE_TOOL_NAMES = new Set(TOOL_REGISTRY.filter((e) => e.category === 'core').map((e) => e.name))
 import { t as i18nT } from '../../i18n'
-import { preprocessMessages, type PreprocessableMessage } from '../preprocessor'
-import {
-  formatMessageCompact,
-  formatToolResultAsText,
-  truncateFormattedMessages,
-  anonymizeMessageNames,
-  countTokens,
-} from '@openchatlab/node-runtime'
+import { applyPreprocessingPipeline, type PreprocessableMessage } from '@openchatlab/node-runtime'
 import { getSkillConfig } from '../skills'
 import { createActivateSkillTool as sharedCreateActivateSkillTool } from '@openchatlab/node-runtime'
 
@@ -62,10 +55,7 @@ function translateTool(tool: AgentTool<any>): AgentTool<any> {
 /**
  * 预处理包装层
  * 拦截工具的 execute 结果：如果 details 中包含 rawMessages，
- * 则执行预处理管道 + 格式化，替换为最终的 LLM 内容
- *
- * 工具约定：返回消息的工具在 details 中放置 rawMessages 字段（结构化消息数组），
- * 处理层负责 preprocess + formatMessageCompact，工具无需感知预处理逻辑。
+ * 则委托共享管道 applyPreprocessingPipeline 执行预处理 + 格式化 + 截断。
  */
 function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): AgentTool<any> {
   const originalExecute = tool.execute
@@ -79,51 +69,22 @@ function wrapWithPreprocessing(tool: AgentTool<any>, context: ToolContext): Agen
         return result
       }
 
-      const raw = details.rawMessages as PreprocessableMessage[]
-      const processed = preprocessMessages(raw, context.preprocessConfig)
+      const { rawMessages, ...restDetails } = details
 
-      let nameMapLine = ''
-      if (context.preprocessConfig?.anonymizeNames) {
-        nameMapLine = anonymizeMessageNames(processed, context.ownerInfo?.platformId)
-      }
-
-      let formatted = processed.map((m) => formatMessageCompact(m, context.locale))
-
-      // Token-aware 截断：超出预算时按策略裁剪消息列表
-      let wasTruncated = false
-      const originalCount = formatted.length
-      if (context.maxToolResultTokens && context.maxToolResultTokens > 0) {
-        const truncResult = truncateFormattedMessages(
-          formatted,
-          context.maxToolResultTokens,
-          TRUNCATION_STRATEGY_MAP.get(tool.name) ?? 'keep_last',
-          countTokens
-        )
-        if (truncResult.wasTruncated) {
-          formatted = truncResult.messages
-          wasTruncated = true
-        }
-      }
-
-      const { rawMessages: _rawMessages, ...restDetails } = details
-      const finalDetails = { ...restDetails, messages: formatted, returned: formatted.length }
-
-      let textContent = formatToolResultAsText(finalDetails)
-
-      if (wasTruncated) {
-        const strategy = TRUNCATION_STRATEGY_MAP.get(tool.name) ?? 'keep_last'
-        const strategyDesc = strategy === 'keep_first' ? 'most relevant' : 'most recent'
-        const notice = `⚠️ Results truncated: ${originalCount} messages found, showing ${formatted.length} ${strategyDesc} due to context limit. Use a narrower time range or more specific keywords for more precise results.`
-        textContent = notice + '\n' + textContent
-      }
-
-      if (nameMapLine) {
-        textContent = nameMapLine + '\n' + textContent
-      }
+      const pipelineResult = applyPreprocessingPipeline({
+        rawMessages: rawMessages as PreprocessableMessage[],
+        preprocessConfig: context.preprocessConfig,
+        locale: context.locale,
+        anonymizeNames: context.preprocessConfig?.anonymizeNames ?? false,
+        ownerPlatformId: context.ownerInfo?.platformId,
+        maxToolResultTokens: context.maxToolResultTokens,
+        truncationStrategy: TRUNCATION_STRATEGY_MAP.get(tool.name) ?? 'keep_last',
+        extraDetails: restDetails as Record<string, unknown>,
+      })
 
       return {
-        content: [{ type: 'text' as const, text: textContent }],
-        details: finalDetails,
+        content: [{ type: 'text' as const, text: pipelineResult.text }],
+        details: pipelineResult.details,
       }
     },
   }

packages/node-runtime/src/ai/index.ts

@@ -51,7 +51,9 @@ export {
   isChineseLocale,
   i18nTexts,
   t,
+  applyPreprocessingPipeline,
 } from './preprocessor'
+export type { PreprocessingPipelineOptions, PreprocessingPipelineResult } from './preprocessor'
 
 // Agent Core
 export type { AgentCoreOptions, AgentCoreEvent, AgentCoreResult, AgentTokenUsage, SimpleHistoryMessage } from './agent'

packages/node-runtime/src/ai/preprocessor/index.ts

@@ -18,3 +18,7 @@ export {
   i18nTexts,
   t,
 } from './format'
+
+// Preprocessing pipeline
+export type { PreprocessingPipelineOptions, PreprocessingPipelineResult } from './preprocessing-pipeline'
+export { applyPreprocessingPipeline } from './preprocessing-pipeline'

packages/node-runtime/src/ai/preprocessor/preprocessing-pipeline.ts

@@ -0,0 +1,82 @@
+/**
+ * 预处理管道（统一实现）
+ *
+ * Electron 端和 Server 端共用的消息预处理管道核心逻辑：
+ * rawMessages → preprocess → [anonymize] → format → truncate → text
+ */
+
+import type { PreprocessConfig, PreprocessableMessage, TruncationStrategy } from './types'
+import { preprocessMessages } from './pipeline'
+import {
+  formatMessageCompact,
+  anonymizeMessageNames,
+  formatToolResultAsText,
+  truncateFormattedMessages,
+} from './format'
+import { countTokens } from '../tokenizer'
+
+export interface PreprocessingPipelineOptions {
+  rawMessages: PreprocessableMessage[]
+  preprocessConfig?: PreprocessConfig
+  locale?: string
+  anonymizeNames?: boolean
+  ownerPlatformId?: string
+  maxToolResultTokens?: number
+  truncationStrategy?: TruncationStrategy
+  /** rawMessages 之外的附加元数据（如 total、timeRange），会合并到输出 details */
+  extraDetails?: Record<string, unknown>
+}
+
+export interface PreprocessingPipelineResult {
+  text: string
+  details: Record<string, unknown>
+}
+
+export function applyPreprocessingPipeline(options: PreprocessingPipelineOptions): PreprocessingPipelineResult {
+  const {
+    rawMessages,
+    preprocessConfig,
+    locale,
+    anonymizeNames = false,
+    ownerPlatformId,
+    maxToolResultTokens,
+    truncationStrategy = 'keep_last',
+    extraDetails = {},
+  } = options
+
+  const processed = preprocessMessages(rawMessages, preprocessConfig)
+
+  let nameMapLine = ''
+  if (anonymizeNames) {
+    nameMapLine = anonymizeMessageNames(processed, ownerPlatformId)
+  }
+
+  let formatted = processed.map((m) => formatMessageCompact(m, locale))
+
+  let wasTruncated = false
+  const originalCount = formatted.length
+
+  if (maxToolResultTokens && maxToolResultTokens > 0) {
+    const truncResult = truncateFormattedMessages(formatted, maxToolResultTokens, truncationStrategy, countTokens)
+    if (truncResult.wasTruncated) {
+      formatted = truncResult.messages
+      wasTruncated = true
+    }
+  }
+
+  const finalDetails: Record<string, unknown> = { ...extraDetails, messages: formatted, returned: formatted.length }
+
+  let textContent = formatToolResultAsText(finalDetails)
+
+  if (wasTruncated) {
+    const strategyDesc = truncationStrategy === 'keep_first' ? 'most relevant' : 'most recent'
+    const notice = `⚠️ Results truncated: ${originalCount} messages found, showing ${formatted.length} ${strategyDesc} due to context limit. Use a narrower time range or more specific keywords for more precise results.`
+    textContent = notice + '\n' + textContent
+  }
+
+  if (nameMapLine) {
+    textContent = nameMapLine + '\n' + textContent
+  }
+
+  return { text: textContent, details: finalDetails }
+}

packages/node-runtime/src/index.ts

@@ -66,7 +66,9 @@ export {
   isChineseLocale,
   i18nTexts,
   t,
+  applyPreprocessingPipeline,
 } from './ai'
+export type { PreprocessingPipelineOptions, PreprocessingPipelineResult } from './ai'
 
 export type {
   AIConversation,

packages/server/src/ai/tool-adapter.ts

@@ -9,11 +9,7 @@ import type { ToolDefinition, ToolExecutionContext } from '@openchatlab/tools'
 import { CoreDataProvider } from '@openchatlab/tools'
 import type { DatabaseAdapter } from '@openchatlab/core'
 import {
-  preprocessMessages,
-  formatMessageCompact,
-  truncateFormattedMessages,
-  formatToolResultAsText,
-  countTokens,
+  applyPreprocessingPipeline,
   type AgentTool,
   type AgentToolResult,
   type PreprocessableMessage,
@@ -45,47 +41,6 @@ function convertJsonSchemaToParameters(schema: ToolDefinition['inputSchema']) {
   }
 }
 
-function applyPreprocessing(
-  rawMessages: PreprocessableMessage[],
-  toolName: string,
-  locale?: string,
-  details?: Record<string, unknown>
-): string {
-  const processed = preprocessMessages(rawMessages)
-  let formatted = processed.map((m) => formatMessageCompact(m, locale))
-
-  let wasTruncated = false
-  const originalCount = formatted.length
-  const budget = DEFAULT_MAX_TOOL_RESULT_TOKENS
-
-  let totalTokens = 0
-  for (const line of formatted) {
-    totalTokens += countTokens(line) + 1
-  }
-
-  if (totalTokens > budget) {
-    const strategy = TOOL_TRUNCATION_STRATEGY[toolName] ?? 'keep_last'
-    const truncResult = truncateFormattedMessages(formatted, budget, strategy, countTokens)
-    if (truncResult.wasTruncated) {
-      formatted = truncResult.messages
-      wasTruncated = true
-    }
-  }
-
-  const finalDetails = { ...details, messages: formatted, returned: formatted.length }
-  const { rawMessages: _, ...restDetails } = finalDetails as Record<string, unknown>
-  let textContent = formatToolResultAsText({ ...restDetails, messages: formatted, returned: formatted.length })
-
-  if (wasTruncated) {
-    const strategy = TOOL_TRUNCATION_STRATEGY[toolName] ?? 'keep_last'
-    const strategyDesc = strategy === 'keep_first' ? 'most relevant' : 'most recent'
-    const notice = `⚠️ Results truncated: ${originalCount} messages found, showing ${formatted.length} ${strategyDesc} due to context limit. Use a narrower time range or more specific keywords for more precise results.`
-    textContent = notice + '\n' + textContent
-  }
-
-  return textContent
-}
-
 export function adaptToolsForAgent(
   tools: ToolDefinition[],
   getContext: () => ServerToolContext
@@ -107,13 +62,14 @@ export function adaptToolsForAgent(
         const result = await tool.handler(params, execCtx)
 
         if (result.rawMessages && result.rawMessages.length > 0) {
-          const textContent = applyPreprocessing(
-            result.rawMessages as PreprocessableMessage[],
-            tool.name,
-            ctx.locale,
-            (result.data ?? {}) as Record<string, unknown>
-          )
-          return { content: [{ type: 'text', text: textContent }], details: null }
+          const pipelineResult = applyPreprocessingPipeline({
+            rawMessages: result.rawMessages as PreprocessableMessage[],
+            locale: ctx.locale,
+            maxToolResultTokens: DEFAULT_MAX_TOOL_RESULT_TOKENS,
+            truncationStrategy: TOOL_TRUNCATION_STRATEGY[tool.name] ?? 'keep_last',
+            extraDetails: (result.data ?? {}) as Record<string, unknown>,
+          })
+          return { content: [{ type: 'text', text: pipelineResult.text }], details: null }
         }
 
         return { content: [{ type: 'text', text: result.content }], details: null }