fix(proxy): use the attached marker for orphan tool result media (#8157)

This commit is contained in:
NeeYoonc authored and GitHub committed 2026-10-24 20:54:40 +08:00
1 parent e8ae28b659
commit 7f9cd9e967
2 files changed
+40 -10

No files matched your search

@@ -20,7 +20,8 @@ use crate::proxy::{
},
tool_media::{
chat_file_from_input_file, flush_pending_chat_tool_media, plan_chat_tool_output_media,
queue_chat_tool_output_media, strip_and_clamp_media_from_tool_value, ToolMediaScope,
plan_chat_tool_output_media_with_marker, queue_chat_tool_output_media,
strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER,
TOOL_RESULT_MEDIA_MOVED_MARKER,
},
};
@@ -1213,9 +1214,10 @@ fn flush_pending_orphan_results(
/// 上下文,但不能以 role=tool 的形式发给严格上游。
///
/// 输出里的图片/文件媒体块复用配对路径的提取逻辑:文本 body 内联
/// `TOOL_RESULT_MEDIA_MOVED_MARKER`,媒体本体作为同一条 user 消息的
/// content parts 跟随,避免图片被 base64 序列化成纯文本——模型既看不到
/// 图,token 还会被瞬间撑爆。
/// `TOOL_RESULT_MEDIA_ATTACHED_MARKER` 占位,媒体本体作为同一条 user
/// 消息的 content parts 跟随,避免图片被 base64 序列化成纯文本——模型既
/// 看不到图,token 还会被瞬间撑爆。配对路径的 "moved to the following
/// user message" 占位在本路径会指向一条不存在的消息,不能沿用。
fn append_orphan_tool_output_as_user_message(messages: &mut Vec<Value>, item: &Value) {
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
let (body, media_parts) = plan_orphan_tool_output_body(item.get("output"));
@@ -1239,10 +1241,16 @@ fn append_orphan_tool_output_as_user_message(messages: &mut Vec<Value>, item: &V
}
/// 计算孤儿工具结果的文本 body 与媒体 parts。媒体识别复用配对路径的
/// `plan_chat_tool_output_media`;没有媒体时保持旧版逐字段的纯文本序列化,
/// 不给无媒体输出多加一层 JSON 包装。
/// `plan_chat_tool_output_media_with_marker`;没有媒体时保持旧版逐字段的
/// 纯文本序列化,不给无媒体输出多加一层 JSON 包装。
///
/// 孤儿路径的媒体就挂在这条 user 消息自己的后续 parts 里,占位符用
/// `TOOL_RESULT_MEDIA_ATTACHED_MARKER`(与 Anthropic/Gemini/Responses
/// bridge 的"媒体已作为原生媒体挂载"用词一致)。
fn plan_orphan_tool_output_body(output: Option<&Value>) -> (String, Vec<Value>) {
if let Some(plan) = output.cloned().and_then(plan_chat_tool_output_media) {
if let Some(plan) = output.cloned().and_then(|output| {
plan_chat_tool_output_media_with_marker(output, TOOL_RESULT_MEDIA_ATTACHED_MARKER)
}) {
return (plan.tool_content, plan.media_parts);
}
let body = match output {
@@ -2996,6 +3004,14 @@ mod tests {
let text = parts[0]["text"].as_str().unwrap();
assert!(text.starts_with(ORPHAN_TOOL_OUTPUT_MARKER), "{text}");
assert!(text.contains("screenshot follows"), "{text}");
assert!(
text.contains(TOOL_RESULT_MEDIA_ATTACHED_MARKER),
"inline media placeholder must match where the media lands: {text}"
);
assert!(
!text.contains("following user message"),
"media travels in this same message: {text}"
);
assert!(
!text.contains(&data_url),
"base64 image must not stay in the text body: {text}"
+17 -3
View File
@@ -57,11 +57,25 @@ impl ToolMediaScope {
/// Scalar strings remain scalar strings after replacement. This matters for a
/// raw image data URL and for JSON encoded inside a tool-output string: adding
/// another layer of JSON string quotes would change what the model sees.
pub(crate) fn plan_chat_tool_output_media(mut output: Value) -> Option<ChatToolOutputMediaPlan> {
///
/// The placeholder says the media was "moved to the following user message".
/// Callers that re-attach the extracted media inside the same message must use
/// [`plan_chat_tool_output_media_with_marker`] so the placeholder matches where
/// the media actually lands.
pub(crate) fn plan_chat_tool_output_media(output: Value) -> Option<ChatToolOutputMediaPlan> {
plan_chat_tool_output_media_with_marker(output, TOOL_RESULT_MEDIA_MOVED_MARKER)
}
/// Same extraction as [`plan_chat_tool_output_media`], with a caller-chosen
/// placeholder text for the stripped media blocks.
pub(crate) fn plan_chat_tool_output_media_with_marker(
mut output: Value,
replacement_marker: &str,
) -> Option<ChatToolOutputMediaPlan> {
let output_was_string = output.is_string();
let replacement_block = json!({
"type": "text",
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
"text": replacement_marker
});
let mut media_parts = Vec::new();
let replaced = strip_and_clamp_media_from_tool_value(
@@ -69,7 +83,7 @@ pub(crate) fn plan_chat_tool_output_media(mut output: Value) -> Option<ChatToolO
&mut media_parts,
ToolMediaScope::AllSupported,
&replacement_block,
TOOL_RESULT_MEDIA_MOVED_MARKER,
replacement_marker,
);
if replaced == 0 {
return None;