diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index 4e1020ac0..0646593a6 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -52,6 +52,14 @@ pub struct HttpBackendConfig { pub extra_headers: BTreeMap, /// Default top-level request fields, applied only when the request omits the key. pub extra_body: BTreeMap, + /// Reasoning effort forced onto every outbound request for this target. + pub reasoning_effort_override: Option, + /// Whether Responses custom tools are bridged through upstream function tools. + pub bridge_custom_tools: bool, + /// Whether Responses deferred tools are made eagerly available upstream. + pub eager_load_tool_search: bool, + /// Whether cross-provider Responses fields are normalized for xAI. + pub xai_responses_compatibility: bool, /// Additional attempts after the initial upstream request. pub max_retries: u32, } @@ -64,6 +72,13 @@ impl fmt::Debug for HttpBackendConfig { .field("forward_auth", &self.forward_auth) .field("extra_header_names", &self.extra_headers.keys()) .field("extra_body_keys", &self.extra_body.keys()) + .field("reasoning_effort_override", &self.reasoning_effort_override) + .field("bridge_custom_tools", &self.bridge_custom_tools) + .field("eager_load_tool_search", &self.eager_load_tool_search) + .field( + "xai_responses_compatibility", + &self.xai_responses_compatibility, + ) .field("max_retries", &self.max_retries) .finish() } @@ -248,6 +263,26 @@ impl Backend { &self.config().extra_body } + /// Target-specific reasoning effort that replaces a caller-supplied value. + pub fn reasoning_effort_override(&self) -> Option<&str> { + self.config().reasoning_effort_override.as_deref() + } + + /// Whether Responses custom tools must be represented as function tools upstream. + pub fn bridge_custom_tools(&self) -> bool { + self.config().bridge_custom_tools + } + + /// Whether Responses tool discovery must be converted to eager definitions upstream. + pub fn eager_load_tool_search(&self) -> bool { + self.config().eager_load_tool_search + } + + /// Whether Responses requests need xAI cross-provider compatibility normalization. + pub fn xai_responses_compatibility(&self) -> bool { + self.config().xai_responses_compatibility + } + /// Additional attempts allowed after the initial request. pub fn max_retries(&self) -> u32 { self.config().max_retries @@ -343,6 +378,10 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort_override: None, + bridge_custom_tools: false, + eager_load_tool_search: false, + xai_responses_compatibility: false, max_retries: 0, } } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 2751ccfd4..71ece79d1 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -4,7 +4,7 @@ //! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral //! request, call the configured backend over HTTP, decode the neutral response. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::time::{Duration, SystemTime}; use async_trait::async_trait; @@ -25,6 +25,14 @@ use crate::backend::Backend; use crate::error::{LlmClientError, Result}; use crate::metrics; use crate::raw::RawResponse; +use crate::responses_custom_tool_bridge::{ + bridge_responses_custom_tool_request, bridge_responses_custom_tool_response, + bridge_responses_custom_tool_stream, responses_custom_tool_names, +}; +use crate::responses_tool_compat::{ + bridge_responses_web_search_to_anthropic, eager_load_responses_tool_search, + normalize_xai_responses_request, responses_web_search_tool, +}; // Headers this client owns or that are hop-by-hop. Backends apply an explicitly // enabled caller credential after generic metadata forwarding skips these. @@ -204,17 +212,42 @@ impl TranslatingLlmClient { model: &ModelId, endpoint: UpstreamEndpoint, ) -> Result { + let responses_web_search = matches!(backend, Backend::Anthropic(_)) + .then(|| responses_web_search_tool(&llm_request)) + .flatten(); let mut body = encode_request(&llm_request, wire_format) .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; // `encode_request` round-trips a preserved same-format body verbatim, // which keeps the caller's original `model`; force the resolved model so // the upstream always sees the target id. set_json_model(&mut body, model); + if matches!(endpoint, UpstreamEndpoint::Completion) { + apply_reasoning_effort_override( + &mut body, + wire_format, + backend.reasoning_effort_override(), + ); + } // Strip before `merge_extra_body` so a target can reinstate either field // deliberately via `extra_body`. if matches!(backend, Backend::Anthropic(_)) { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); + if let Some(tool) = &responses_web_search { + bridge_responses_web_search_to_anthropic(&mut body, tool); + } + } + if matches!(backend, Backend::OpenAiResponses(_)) { + sanitize_openai_responses_replay_items(&mut body); + if backend.eager_load_tool_search() { + eager_load_responses_tool_search(&mut body); + } + if backend.xai_responses_compatibility() { + normalize_xai_responses_request(&mut body); + } + if backend.bridge_custom_tools() { + bridge_responses_custom_tool_request(&mut body); + } } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { @@ -411,6 +444,11 @@ impl TranslatingLlmClient { message: format!("model {model_id:?} has no backend for format {wire_format}"), } })?; + let bridged_custom_tools = if backend.bridge_custom_tools() { + responses_custom_tool_names(&llm_request) + } else { + HashSet::new() + }; let http_response = self .send_encoded( @@ -441,12 +479,17 @@ impl TranslatingLlmClient { }) }); let chunks = decode_stream(bytes, wire_format)?; + let chunks = + bridge_responses_custom_tool_stream(chunks, wire_format, bridged_custom_tools); LlmResponse::Stream(chunks) } EncodedResponse::Buffered { body, .. } => { - let body = serde_json::from_slice::(&body).map_err(|error| { + let mut body = serde_json::from_slice::(&body).map_err(|error| { LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}")) })?; + if wire_format == WireFormat::OpenAiResponses && !bridged_custom_tools.is_empty() { + bridge_responses_custom_tool_response(&mut body, &bridged_custom_tools); + } let agg = decode_aggregated_response(&body, wire_format) .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; LlmResponse::Agg(agg) @@ -674,6 +717,54 @@ fn set_json_model(body: &mut Value, model: &str) { } } +// Replaces the provider-shaped effort field after encoding so the override also +// applies when same-format request preservation would otherwise retain the +// caller's original wire body. +fn apply_reasoning_effort_override( + body: &mut Value, + wire_format: WireFormat, + effort: Option<&str>, +) { + let (Value::Object(object), Some(effort)) = (body, effort) else { + return; + }; + match wire_format { + WireFormat::OpenAiChat => { + object.insert( + "reasoning_effort".to_string(), + Value::String(effort.to_string()), + ); + } + WireFormat::OpenAiResponses => { + let reasoning = object + .entry("reasoning".to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !reasoning.is_object() { + *reasoning = Value::Object(Map::new()); + } + reasoning + .as_object_mut() + .expect("reasoning was normalized to an object") + .insert("effort".to_string(), Value::String(effort.to_string())); + } + WireFormat::AnthropicMessages => { + object + .entry("thinking".to_string()) + .or_insert_with(|| serde_json::json!({"type": "adaptive"})); + let output_config = object + .entry("output_config".to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !output_config.is_object() { + *output_config = Value::Object(Map::new()); + } + output_config + .as_object_mut() + .expect("output_config was normalized to an object") + .insert("effort".to_string(), Value::String(effort.to_string())); + } + } +} + // Drops fields accepted by OpenAI-like APIs but rejected by Anthropic Messages. // // A router can serve earlier turns of a session from an OpenAI-format target and @@ -688,6 +779,75 @@ fn strip_anthropic_incompatible_fields(body: &mut Value) { } } +// Normalizes response items that coding agents replay as Responses input. +// Responses providers reject response-only `status` metadata, malformed or +// provider-bound hosted-tool records, non-empty reasoning `content`, and encrypted +// reasoning created by a different provider. Ciphertext and tool provenance are not +// represented on replay items, so dynamic routes retain portable summaries, +// assistant messages, and `call_id` linkage while discarding provider-bound data. +fn sanitize_openai_responses_replay_items(body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| { + let Value::Object(item) = item else { + return true; + }; + match item.get("type").and_then(Value::as_str) { + // A hosted search is represented by its provider-specific call item + // followed by a portable assistant message containing the answer and + // citations. Replaying the call itself to another provider is both + // unnecessary and rejected by OpenAI-compatible local backends. + Some("web_search_call") => return false, + Some("message") => { + item.remove("status"); + strip_invalid_openai_responses_replay_id(item); + } + Some("reasoning") => { + item.remove("status"); + item.remove("encrypted_content"); + strip_invalid_openai_responses_replay_id(item); + if item.contains_key("content") { + item.insert("content".to_string(), Value::Array(Vec::new())); + } + } + Some("function_call") | Some("function_call_output") => { + // Function item IDs are optional replay metadata. `call_id` + // carries the portable call/result relationship. + item.remove("status"); + strip_invalid_openai_responses_replay_id(item); + } + Some("custom_tool_call") | Some("custom_tool_call_output") => { + item.remove("status"); + // Custom-tool item IDs use provider-specific namespaces (for + // example OpenAI requires `ctc...`, while a function-tool bridge + // can originate `fc_...`). They are optional on replay; `call_id` + // is the portable call/result relationship. + item.remove("id"); + } + _ => {} + } + true + }); +} + +// Replay IDs are optional, so discard cross-provider values OpenAI cannot parse. +fn strip_invalid_openai_responses_replay_id(item: &mut Map) { + let Some(id) = item.get("id") else { + return; + }; + let valid = id.as_str().is_some_and(|id| { + !id.is_empty() + && id.len() <= 64 + && id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + }); + if !valid { + item.remove("id"); + } +} + // Removes replayed `thinking` blocks that carry no signature. // // Anthropic requires signed thinking blocks on replay. A router can serve earlier @@ -832,6 +992,10 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort_override: None, + bridge_custom_tools: false, + eager_load_tool_search: false, + xai_responses_compatibility: false, max_retries: 0, } } @@ -869,6 +1033,48 @@ mod tests { )] } + fn responses_map(base_url: &str) -> Vec { + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(config(base_url)), + None, + )] + } + + fn responses_map_with_custom_tool_bridge(base_url: &str) -> Vec { + let mut backend = config(base_url); + backend.bridge_custom_tools = true; + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + )] + } + + fn responses_map_with_tool_compat(base_url: &str) -> Vec { + let mut backend = config(base_url); + backend.bridge_custom_tools = true; + backend.eager_load_tool_search = true; + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + )] + } + + fn responses_map_with_reasoning_effort_override( + base_url: &str, + effort: &str, + ) -> Vec { + let mut backend = config(base_url); + backend.reasoning_effort_override = Some(effort.to_string()); + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + )] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -973,6 +1179,7 @@ mod tests { request } + // Exercises the complete preserved request/response bridge for a buffered call. #[tokio::test] async fn missing_model_errors() -> std::result::Result<(), Box> { @@ -990,6 +1197,7 @@ mod tests { Ok(()) } + // Streamed function argument JSON is withheld until it can be restored as raw custom input. #[tokio::test] async fn unknown_model_errors() -> std::result::Result<(), Box> { @@ -1103,6 +1311,574 @@ mod tests { Ok(()) } + #[test] + fn responses_replay_sanitizer_drops_overlong_optional_item_ids() { + let overlong_id = format!("msg_{}", "a".repeat(80)); + let mut body = json!({ + "input": [ + {"type": "message", "id": overlong_id, "role": "assistant", "content": []}, + {"type": "reasoning", "id": overlong_id, "summary": [], "content": []}, + { + "type": "function_call", + "id": overlong_id, + "call_id": "call_1", + "name": "echo", + "arguments": "{}" + }, + { + "type": "function_call_output", + "id": overlong_id, + "call_id": "call_1", + "output": "OK" + } + ] + }); + + sanitize_openai_responses_replay_items(&mut body); + + assert!( + body["input"] + .as_array() + .is_some_and(|items| { items.iter().all(|item| item.get("id").is_none()) }) + ); + assert_eq!(body["input"][2]["call_id"], "call_1"); + assert_eq!(body["input"][3]["call_id"], "call_1"); + } + + #[tokio::test] + async fn responses_backend_sanitizes_replayed_response_fields_before_sending() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let body = json!({ + "model": "ctm-auto", + "input": [ + { + "type": "message", + "id": "msg_valid-0", + "role": "user", + "content": "start" + }, + { + "type": "reasoning", + "id": "rs_1", + "status": "completed", + "content": [{ + "type": "reasoning_text", + "text": "private provider reasoning" + }], + "summary": [{ + "type": "summary_text", + "text": "Portable reasoning summary." + }], + "encrypted_content": "opaque" + }, + { + "type": "message", + "id": "{\"v\":1}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "working"}] + }, + { + "type": "custom_tool_call", + "id": "fc_cross_provider", + "call_id": "call_1", + "name": "apply_patch", + "input": "*** Begin Patch\n*** End Patch", + "status": "completed" + }, + { + "type": "custom_tool_call_output", + "id": "cto_cross_provider", + "call_id": "call_1", + "output": "Done", + "status": "completed" + }, + { + "type": "function_call", + "id": "fc_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_too_long", + "call_id": "call_2", + "name": "echo", + "arguments": "{\"value\":\"OK\"}", + "status": "completed" + }, + { + "type": "function_call_output", + "id": "fco_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_too_long", + "call_id": "call_2", + "output": "OK", + "status": "completed" + }, + { + "type": "web_search_call", + "id": "ws_cross_provider", + "status": "completed", + "action": { + "type": "search", + "query": "current Rust release" + } + }, + {"type": "message", "role": "user", "content": "continue"} + ] + }); + + client + .call_rewrite_model_raw( + body, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + + let requests = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let forwarded: Value = serde_json::from_slice(&requests[0].body)?; + assert_eq!(forwarded["model"], "gpt"); + assert_eq!(forwarded["input"][0]["id"], "msg_valid-0"); + assert_eq!(forwarded["input"][1].get("status"), None); + assert_eq!(forwarded["input"][1]["id"], "rs_1"); + assert_eq!(forwarded["input"][1]["content"], json!([])); + assert_eq!(forwarded["input"][1].get("encrypted_content"), None); + assert_eq!( + forwarded["input"][1]["summary"][0]["text"], + "Portable reasoning summary." + ); + assert_eq!(forwarded["input"][2].get("status"), None); + assert_eq!(forwarded["input"][2].get("id"), None); + assert_eq!( + forwarded["input"][2]["content"], + json!([{"type": "output_text", "text": "working"}]) + ); + assert_eq!(forwarded["input"][3].get("id"), None); + assert_eq!(forwarded["input"][3].get("status"), None); + assert_eq!(forwarded["input"][3]["call_id"], "call_1"); + assert_eq!(forwarded["input"][4].get("id"), None); + assert_eq!(forwarded["input"][4].get("status"), None); + assert_eq!(forwarded["input"][4]["call_id"], "call_1"); + assert_eq!(forwarded["input"][5].get("id"), None); + assert_eq!(forwarded["input"][5].get("status"), None); + assert_eq!(forwarded["input"][5]["call_id"], "call_2"); + assert_eq!(forwarded["input"][6].get("id"), None); + assert_eq!(forwarded["input"][6].get("status"), None); + assert_eq!(forwarded["input"][6]["call_id"], "call_2"); + assert_eq!(forwarded["input"].as_array().map(Vec::len), Some(8)); + assert!(forwarded["input"].as_array().is_some_and(|items| { + items + .iter() + .all(|item| item.get("type").and_then(Value::as_str) != Some("web_search_call")) + })); + assert_eq!(forwarded["input"][7]["content"], "continue"); + Ok(()) + } + + #[tokio::test] + async fn target_reasoning_effort_override_replaces_preserved_responses_effort() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map_with_reasoning_effort_override( + &format!("{}/v1", server.uri()), + "max", + ))?; + let body = json!({ + "model": "ctm-auto", + "input": "inspect the image", + "reasoning": {"effort": "low", "summary": "auto"} + }); + + client + .call_rewrite_model_raw( + body, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + + let requests = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let forwarded: Value = serde_json::from_slice(&requests[0].body)?; + assert_eq!(forwarded["reasoning"]["effort"], "max"); + assert_eq!(forwarded["reasoning"]["summary"], "auto"); + Ok(()) + } + + #[tokio::test] + async fn responses_custom_tool_bridge_round_trips_buffered_calls() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "function_call", + "id": "fc_1", + "call_id": "call_2", + "name": "apply_patch", + "arguments": "{\"input\":\"*** Begin Patch\\n*** End Patch\"}", + "status": "completed" + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map_with_custom_tool_bridge(&format!( + "{}/v1", + server.uri() + )))?; + let raw = json!({ + "model": "ctm-auto", + "input": [ + {"role": "user", "content": "Continue the research edit."}, + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_1", + "name": "apply_patch", + "input": "*** Begin Patch\n*** End Patch" + }, + { + "type": "custom_tool_call_output", + "call_id": "call_1", + "output": "Done" + } + ], + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "Edit files", + "format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"} + }], + "tool_choice": {"type": "custom", "name": "apply_patch"} + }); + + let RawResponse::Buffered(body) = client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await? + else { + panic!("expected a buffered response"); + }; + + let requests = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let forwarded: Value = serde_json::from_slice(&requests[0].body)?; + assert_eq!(forwarded["tools"][0]["type"], "function"); + assert_eq!(forwarded["tool_choice"]["type"], "function"); + assert_eq!(forwarded["input"][1]["type"], "function_call"); + assert_eq!(forwarded["input"][1].get("id"), None); + assert_eq!(forwarded["input"][2]["type"], "function_call_output"); + assert_eq!(body["output"][0]["type"], "custom_tool_call"); + assert_eq!(body["output"][0]["input"], "*** Begin Patch\n*** End Patch"); + assert_eq!(body["output"][0].get("id"), None); + assert_eq!(body["model"], "gpt"); + Ok(()) + } + + #[tokio::test] + async fn responses_tool_search_bridge_eagerly_exposes_deferred_tools() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map_with_tool_compat(&format!( + "{}/v1", + server.uri() + )))?; + let raw = json!({ + "model": "ctm-auto", + "input": [ + {"role": "user", "content": "Continue."}, + { + "type": "tool_search_call", + "execution": "client", + "call_id": "search_1", + "status": "completed", + "arguments": {"goal": "Find an editing tool."} + }, + { + "type": "tool_search_output", + "execution": "client", + "call_id": "search_1", + "status": "completed", + "tools": [{ + "type": "function", + "name": "write_file", + "defer_loading": true, + "parameters": {"type": "object"} + }] + } + ], + "tools": [ + { + "type": "namespace", + "name": "repo", + "tools": [{ + "type": "function", + "name": "read_file", + "defer_loading": true, + "parameters": {"type": "object"} + }] + }, + { + "type": "custom", + "name": "apply_patch", + "description": "Edit files", + "defer_loading": true, + "format": {"type": "grammar", "syntax": "regex", "definition": ".+"} + }, + {"type": "web_search"}, + {"type": "tool_search"} + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + + let requests = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let forwarded: Value = serde_json::from_slice(&requests[0].body)?; + let tools = forwarded["tools"] + .as_array() + .ok_or("forwarded tools should be an array")?; + assert!(tools.iter().all(|tool| { + !matches!( + tool.get("type").and_then(Value::as_str), + Some("namespace" | "tool_search") + ) && tool.get("defer_loading").is_none() + })); + assert!(tools.iter().any(|tool| { + tool.get("type").and_then(Value::as_str) == Some("function") + && tool.get("name").and_then(Value::as_str) == Some("apply_patch") + })); + assert!( + tools + .iter() + .any(|tool| { tool.get("type").and_then(Value::as_str) == Some("web_search") }) + ); + assert!(forwarded["input"].as_array().is_some_and(|input| { + input.iter().all(|item| { + !matches!( + item.get("type").and_then(Value::as_str), + Some("tool_search_call" | "tool_search_output") + ) + }) + })); + Ok(()) + } + + #[tokio::test] + async fn responses_web_search_translates_to_anthropic_server_tool() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/messages")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "searched"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&anthropic_map(&server.uri()))?; + let raw = json!({ + "model": "ctm-auto", + "input": "Search current sources.", + "tools": [{ + "type": "web_search", + "filters": {"allowed_domains": ["example.com"]}, + "user_location": {"type": "approximate", "country": "US"} + }] + }); + + let RawResponse::Buffered(body) = client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("claude")), + WireFormat::OpenAiResponses, + ) + .await? + else { + panic!("expected a buffered response"); + }; + + let requests = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let forwarded: Value = serde_json::from_slice(&requests[0].body)?; + assert_eq!(forwarded["tools"][0]["type"], "web_search_20250305"); + assert_eq!(forwarded["tools"][0]["name"], "web_search"); + assert_eq!( + forwarded["tools"][0]["allowed_domains"], + json!(["example.com"]) + ); + assert_eq!(forwarded["tools"][0]["user_location"]["country"], "US"); + assert_eq!(body["output"][0]["content"][0]["text"], "searched"); + Ok(()) + } + + #[tokio::test] + async fn responses_custom_tool_bridge_round_trips_streamed_calls() + -> std::result::Result<(), Box> { + use futures::TryStreamExt; + + let server = MockServer::start().await; + let arguments = r#"{"input":"*** Begin Patch\n*** End Patch"}"#; + let sse = format!( + "data: {{\"type\":\"response.created\",\"response\":{{\"id\":\"resp_1\",\"model\":\"gpt\",\"status\":\"in_progress\",\"output\":[]}}}}\n\n\ + data: {{\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"apply_patch\",\"arguments\":\"\"}}}}\n\n\ + data: {{\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"item_id\":\"fc_1\",\"delta\":{arguments:?}}}\n\n\ + data: {{\"type\":\"response.function_call_arguments.done\",\"output_index\":0,\"item_id\":\"fc_1\",\"arguments\":{arguments:?}}}\n\n\ + data: {{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"apply_patch\",\"arguments\":{arguments:?},\"status\":\"completed\"}}}}\n\n\ + data: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_1\",\"model\":\"gpt\",\"status\":\"completed\",\"output\":[{{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"apply_patch\",\"arguments\":{arguments:?},\"status\":\"completed\"}}],\"usage\":{{}}}}}}\n\n\ + data: [DONE]\n\n" + ); + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&responses_map_with_custom_tool_bridge(&format!( + "{}/v1", + server.uri() + )))?; + let raw = json!({ + "model": "ctm-auto", + "input": "Make the requested research edit.", + "stream": true, + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "Edit files", + "format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"} + }], + "tool_choice": {"type": "custom", "name": "apply_patch"} + }); + + let RawResponse::Stream(stream) = client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await? + else { + panic!("expected a streamed response"); + }; + let events: Vec = stream.try_collect().await?; + + assert!(!events.iter().any(|event| { + event.get("type").and_then(Value::as_str) + == Some("response.function_call_arguments.delta") + })); + assert!(events.iter().any(|event| { + event.get("type").and_then(Value::as_str) + == Some("response.custom_tool_call_input.done") + && event.get("input").and_then(Value::as_str) + == Some("*** Begin Patch\n*** End Patch") + })); + for event_type in ["response.output_item.added", "response.output_item.done"] { + assert!(events.iter().any(|event| { + event.get("type").and_then(Value::as_str) == Some(event_type) + && event["item"]["type"] == "custom_tool_call" + && event["item"].get("id").is_none() + })); + } + let completed = events + .iter() + .find(|event| event.get("type").and_then(Value::as_str) == Some("response.completed")) + .ok_or("missing response.completed")?; + assert_eq!( + completed["response"]["output"][0]["type"], + "custom_tool_call" + ); + assert_eq!(completed["response"]["output"][0].get("id"), None); + assert_eq!(completed["response"]["model"], "gpt"); + Ok(()) + } + #[tokio::test] async fn invalid_json_is_a_response_translation_error() -> std::result::Result<(), Box> { diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index e10a17dd6..9bb1cee3f 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -23,12 +23,17 @@ pub mod metrics; mod observability; mod observation; pub mod raw; +mod responses_custom_tool_bridge; +mod responses_tool_compat; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; pub use client::{ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; -pub use observation::{LlmCallObservation, RunObservation, RunObserver}; +pub use observation::{ + ClassifierContentObservation, LlmCallObservation, LlmCallStartObservation, RunObservation, + RunObserver, +}; pub use raw::RawResponse; -pub use run::{ClientRouter, run}; +pub use run::{ClientRouter, ObservationConfig, run, run_with_observation_config}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs index c02475b4b..b20522f95 100644 --- a/crates/libsy-llm-client/src/observability.rs +++ b/crates/libsy-llm-client/src/observability.rs @@ -174,6 +174,7 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> { LibsyError::ClientCall { source, .. } => llm_client_error_type(source), LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"), LibsyError::NoTargets => Cow::Borrowed("no_targets"), + LibsyError::NoCompatibleTargets { .. } => Cow::Borrowed("no_compatible_targets"), LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"), LibsyError::Driver(_) => Cow::Borrowed("driver_error"), LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"), diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs index cff08fff4..411f04e1b 100644 --- a/crates/libsy-llm-client/src/observation.rs +++ b/crates/libsy-llm-client/src/observation.rs @@ -6,7 +6,33 @@ use std::sync::Arc; use std::time::Duration; -use switchyard_protocol::{ModelId, Usage}; +use switchyard_protocol::{Decision, LlmRequest, ModelId, Usage}; + +/// Prompt and model-produced content from one non-answer classifier or judge call. +#[derive(Clone, Debug)] +pub struct ClassifierContentObservation { + /// Model that produced the routing verdict. + pub selected_model: ModelId, + /// Exact normalized request sent to the classifier target, excluding transport headers. + pub request: LlmRequest, + /// Model-produced reasoning content, when the provider returned it separately. + pub reasoning: Option, + /// Text verdict consumed by the routing policy, including invalid replies. + pub verdict: Option, + /// Whether the provider call itself completed successfully. + pub is_success: bool, + /// Time spent waiting for the classifier call to resolve. + pub duration: Duration, +} + +/// One model call observed immediately before it is sent to its routed client. +#[derive(Clone, Debug)] +pub struct LlmCallStartObservation { + /// Model selected for the call. + pub selected_model: ModelId, + /// Whether this call generates an answer rather than a routing verdict. + pub is_answer_call: bool, +} /// One completed model call observed at the algorithm offload boundary. #[derive(Clone, Debug)] @@ -26,6 +52,12 @@ pub struct LlmCallObservation { /// One request-scoped observation emitted by the algorithm runner. #[derive(Clone, Debug)] pub enum RunObservation { + /// A routing decision, observed before its answer call starts. + RoutingDecision(Decision), + /// A model call about to start. + LlmCallStarted(LlmCallStartObservation), + /// Prompt, reasoning, and verdict from a classifier or judge call. + ClassifierContent(ClassifierContentObservation), /// A completed model call. LlmCall(LlmCallObservation), /// Routing time recorded by the `switchyard.routing_overhead_ms` metric. diff --git a/crates/libsy-llm-client/src/responses_custom_tool_bridge.rs b/crates/libsy-llm-client/src/responses_custom_tool_bridge.rs new file mode 100644 index 000000000..76b6a400b --- /dev/null +++ b/crates/libsy-llm-client/src/responses_custom_tool_bridge.rs @@ -0,0 +1,424 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compatibility bridge for Responses providers that support function tools but not custom tools. + +use std::collections::HashSet; + +use futures_util::StreamExt; +use futures_util::future::ready; +use serde_json::{Map, Value, json}; +use switchyard_protocol::{LlmRequest, LlmResponseStream, LlmResponseStreamEvent, WireFormat}; + +/// Returns custom tool names retained in the original Responses request. +pub(crate) fn responses_custom_tool_names(request: &LlmRequest) -> HashSet { + let format = WireFormat::OpenAiResponses.into(); + request + .preservation + .requests + .get(&format) + .and_then(|body| body.get("tools")) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tool| { + (tool.get("type").and_then(Value::as_str) == Some("custom")) + .then(|| tool.get("name").and_then(Value::as_str)) + .flatten() + .filter(|name| !name.is_empty()) + .map(ToOwned::to_owned) + }) + .collect() +} + +/// Converts custom definitions and replay items to Responses function equivalents. +pub(crate) fn bridge_responses_custom_tool_request(body: &mut Value) { + let Some(object) = body.as_object_mut() else { + return; + }; + let Some(tools) = object.get_mut("tools").and_then(Value::as_array_mut) else { + return; + }; + + let mut names = HashSet::new(); + for tool in tools { + let Some(custom) = tool.as_object() else { + continue; + }; + if custom.get("type").and_then(Value::as_str) != Some("custom") { + continue; + } + let Some(name) = custom + .get("name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + else { + continue; + }; + let name = name.to_string(); + let description = custom + .get("description") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + *tool = json!({ + "type": "function", + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": { + "input": { + "type": "string", + "description": "Raw input for the custom tool." + } + }, + "required": ["input"], + "additionalProperties": false + } + }); + names.insert(name); + } + if names.is_empty() { + return; + } + + if let Some(choice) = object.get_mut("tool_choice").and_then(Value::as_object_mut) + && choice.get("type").and_then(Value::as_str) == Some("custom") + && choice + .get("name") + .and_then(Value::as_str) + .is_some_and(|name| names.contains(name)) + { + choice.insert("type".to_string(), Value::String("function".to_string())); + } + + let Some(input) = object.get_mut("input").and_then(Value::as_array_mut) else { + return; + }; + for item in input { + bridge_responses_custom_tool_replay_item(item, &names); + } +} + +// Rewrites one prior custom call or result into an upstream function replay item. +fn bridge_responses_custom_tool_replay_item(item: &mut Value, names: &HashSet) { + let Some(object) = item.as_object() else { + return; + }; + match object.get("type").and_then(Value::as_str) { + Some("custom_tool_call") + if object + .get("name") + .and_then(Value::as_str) + .is_some_and(|name| names.contains(name)) => + { + let Some(name) = object.get("name").and_then(Value::as_str) else { + return; + }; + let mut replacement = Map::new(); + replacement.insert( + "type".to_string(), + Value::String("function_call".to_string()), + ); + replacement.insert("name".to_string(), Value::String(name.to_string())); + if let Some(call_id) = object.get("call_id").and_then(Value::as_str) { + replacement.insert("call_id".to_string(), Value::String(call_id.to_string())); + } + if let Some(id) = object + .get("id") + .and_then(Value::as_str) + .filter(|id| id.starts_with("fc_")) + { + replacement.insert("id".to_string(), Value::String(id.to_string())); + } + let input = object + .get("input") + .and_then(Value::as_str) + .unwrap_or_default(); + replacement.insert( + "arguments".to_string(), + json!({"input": input}).to_string().into(), + ); + *item = Value::Object(replacement); + } + Some("custom_tool_call_output") => { + let mut replacement = Map::new(); + replacement.insert( + "type".to_string(), + Value::String("function_call_output".to_string()), + ); + if let Some(call_id) = object.get("call_id").and_then(Value::as_str) { + replacement.insert("call_id".to_string(), Value::String(call_id.to_string())); + } + if let Some(output) = object.get("output") { + replacement.insert("output".to_string(), output.clone()); + } + *item = Value::Object(replacement); + } + _ => {} + } +} + +/// Converts bridged function calls in a buffered Responses result back to custom calls. +pub(crate) fn bridge_responses_custom_tool_response(body: &mut Value, names: &HashSet) { + bridge_response_output(body.get_mut("output"), names); + bridge_response_output( + body.get_mut("response") + .and_then(|value| value.get_mut("output")), + names, + ); +} + +// Converts function-call items in one Responses output array. +fn bridge_response_output(output: Option<&mut Value>, names: &HashSet) { + let Some(output) = output.and_then(Value::as_array_mut) else { + return; + }; + for item in output { + bridge_function_call_item(item, names); + } +} + +// Converts one bridged function-call output item to the caller's custom-tool shape. +fn bridge_function_call_item(item: &mut Value, names: &HashSet) -> bool { + let Some(object) = item.as_object_mut() else { + return false; + }; + if object.get("type").and_then(Value::as_str) != Some("function_call") + || !object + .get("name") + .and_then(Value::as_str) + .is_some_and(|name| names.contains(name)) + { + return false; + } + let input = custom_input_from_arguments(object.get("arguments")); + object.insert( + "type".to_string(), + Value::String("custom_tool_call".to_string()), + ); + object.insert("input".to_string(), Value::String(input)); + object.remove("arguments"); + object.remove("status"); + // `fc_...` identifies an upstream function-call item and is invalid for a + // custom-tool item (`ctc...` on OpenAI). The ID is optional; `call_id` retains + // the portable tool-call identity used to associate the result. + object.remove("id"); + true +} + +// Extracts the raw custom input from the JSON argument wrapper used upstream. +fn custom_input_from_arguments(arguments: Option<&Value>) -> String { + let Some(arguments) = arguments else { + return String::new(); + }; + match arguments { + Value::String(raw) => serde_json::from_str::(raw) + .ok() + .and_then(|value| custom_input_from_value(&value)) + .unwrap_or_else(|| raw.clone()), + value => custom_input_from_value(value).unwrap_or_else(|| value.to_string()), + } +} + +// Accepts the bridge's canonical key plus common coding-agent aliases. +fn custom_input_from_value(value: &Value) -> Option { + if let Some(value) = value.as_str() { + return Some(value.to_string()); + } + let object = value.as_object()?; + for key in ["input", "arguments", "patch"] { + if let Some(value) = object.get(key).and_then(Value::as_str) { + return Some(value.to_string()); + } + } + if object.len() == 1 { + return object + .values() + .next() + .and_then(Value::as_str) + .map(ToOwned::to_owned); + } + None +} + +/// Rewrites preserved Responses stream events while retaining their normalized metadata. +pub(crate) fn bridge_responses_custom_tool_stream( + stream: LlmResponseStream, + wire_format: WireFormat, + names: HashSet, +) -> LlmResponseStream { + if wire_format != WireFormat::OpenAiResponses || names.is_empty() { + return stream; + } + let mut bridged_indices = HashSet::new(); + Box::pin(stream.filter_map(move |item| { + let rewritten = match item { + Err(error) => Some(Err(error)), + Ok(event) => bridge_stream_event(event, &names, &mut bridged_indices).map(Ok), + }; + ready(rewritten) + })) +} + +// Converts or suppresses one preserved upstream function-call stream event. +fn bridge_stream_event( + event: LlmResponseStreamEvent, + names: &HashSet, + bridged_indices: &mut HashSet, +) -> Option { + let (preservation, normalized) = event.into_parts(); + let Some(preservation) = preservation else { + return Some(LlmResponseStreamEvent::new(normalized)); + }; + let (source, mut raw) = preservation.into_parts(); + if source != WireFormat::OpenAiResponses.into() { + return Some(LlmResponseStreamEvent::preserved(source, raw, normalized)); + } + if !bridge_stream_raw_event(&mut raw, names, bridged_indices) { + return None; + } + Some(LlmResponseStreamEvent::preserved(source, raw, normalized)) +} + +// Returns false for function argument deltas that cannot be exposed as raw custom input. +fn bridge_stream_raw_event( + event: &mut Value, + names: &HashSet, + bridged_indices: &mut HashSet, +) -> bool { + match event.get("type").and_then(Value::as_str) { + Some("response.output_item.added") | Some("response.output_item.done") => { + let index = event + .get("output_index") + .and_then(Value::as_u64) + .and_then(|index| usize::try_from(index).ok()); + if let Some(item) = event.get_mut("item") + && bridge_function_call_item(item, names) + && let Some(index) = index + { + bridged_indices.insert(index); + } + } + Some("response.function_call_arguments.delta") => { + return !stream_event_is_bridged(event, bridged_indices); + } + Some("response.function_call_arguments.done") + if stream_event_is_bridged(event, bridged_indices) => + { + let input = custom_input_from_arguments(event.get("arguments")); + let Some(object) = event.as_object_mut() else { + return true; + }; + object.insert( + "type".to_string(), + Value::String("response.custom_tool_call_input.done".to_string()), + ); + object.insert("input".to_string(), Value::String(input)); + object.remove("arguments"); + } + Some("response.completed") | Some("response.incomplete") => { + bridge_responses_custom_tool_response(event, names); + } + _ => {} + } + true +} + +// Matches argument events to the custom call discovered in output_item.added. +fn stream_event_is_bridged(event: &Value, bridged_indices: &HashSet) -> bool { + event + .get("output_index") + .and_then(Value::as_u64) + .and_then(|index| usize::try_from(index).ok()) + .is_some_and(|index| bridged_indices.contains(&index)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Custom definitions and replay items must all use the same function-call protocol upstream. + #[test] + fn request_bridge_rewrites_custom_definitions_choice_and_replay() { + let mut body = json!({ + "model": "grok", + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "Edit files", + "format": {"type": "grammar", "syntax": "lark", "definition": "start: /.+/"} + }], + "tool_choice": {"type": "custom", "name": "apply_patch"}, + "input": [ + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_1", + "name": "apply_patch", + "input": "*** Begin Patch\n*** End Patch", + "status": "completed" + }, + { + "type": "custom_tool_call_output", + "id": "cto_1", + "call_id": "call_1", + "output": "Done", + "status": "completed" + } + ] + }); + + bridge_responses_custom_tool_request(&mut body); + + assert_eq!(body["tools"][0]["type"], "function"); + assert_eq!(body["tools"][0]["parameters"]["required"], json!(["input"])); + assert_eq!( + body["tool_choice"], + json!({"type": "function", "name": "apply_patch"}) + ); + assert_eq!(body["input"][0]["type"], "function_call"); + assert_eq!(body["input"][0].get("id"), None); + assert_eq!( + body["input"][0]["arguments"], + json!("{\"input\":\"*** Begin Patch\\n*** End Patch\"}") + ); + assert_eq!( + body["input"][1], + json!({ + "type": "function_call_output", + "call_id": "call_1", + "output": "Done" + }) + ); + } + + // Function argument wrappers must not leak into the raw custom input returned to the caller. + #[test] + fn response_bridge_unwraps_function_arguments() { + let names = HashSet::from(["apply_patch".to_string()]); + let mut body = json!({ + "output": [{ + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "apply_patch", + "arguments": "{\"input\":\"*** Begin Patch\\n*** End Patch\"}", + "status": "completed" + }] + }); + + bridge_responses_custom_tool_response(&mut body, &names); + + assert_eq!( + body["output"][0], + json!({ + "type": "custom_tool_call", + "call_id": "call_1", + "name": "apply_patch", + "input": "*** Begin Patch\n*** End Patch" + }) + ); + } +} diff --git a/crates/libsy-llm-client/src/responses_tool_compat.rs b/crates/libsy-llm-client/src/responses_tool_compat.rs new file mode 100644 index 000000000..51b02c971 --- /dev/null +++ b/crates/libsy-llm-client/src/responses_tool_compat.rs @@ -0,0 +1,409 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compatibility transforms for provider-hosted tools received through Responses. + +use serde_json::{Map, Value}; +use switchyard_protocol::{LlmRequest, WireFormat}; + +/// Replaces deferred Responses tool discovery with eagerly available definitions. +/// +/// Providers without `tool_search` can still call the same client tools; they receive every +/// deferred definition up front and do not see the discovery-only replay records. +pub(crate) fn eager_load_responses_tool_search(body: &mut Value) { + let Some(object) = body.as_object_mut() else { + return; + }; + let has_tool_search = object + .get("tools") + .and_then(Value::as_array) + .is_some_and(|tools| tools.iter().any(is_tool_search)); + let has_tool_search_replay = object + .get("input") + .and_then(Value::as_array) + .is_some_and(|input| input.iter().any(is_tool_search_replay)); + if !has_tool_search && !has_tool_search_replay { + return; + } + + let mut eager_tools = Vec::new(); + if let Some(tools) = object.get_mut("tools").and_then(Value::as_array_mut) { + for tool in std::mem::take(tools) { + push_eager_tool(tool, &mut eager_tools); + } + } + + if let Some(input) = object.get_mut("input").and_then(Value::as_array_mut) { + let mut retained = Vec::with_capacity(input.len()); + for mut item in std::mem::take(input) { + match item.get("type").and_then(Value::as_str) { + Some("tool_search_call") => {} + Some("tool_search_output") => { + if let Some(tools) = item.get_mut("tools").and_then(Value::as_array_mut) { + for tool in std::mem::take(tools) { + push_eager_tool(tool, &mut eager_tools); + } + } + } + _ => { + if item.get("type").and_then(Value::as_str) == Some("function_call") + && let Some(item) = item.as_object_mut() + { + item.remove("namespace"); + } + retained.push(item); + } + } + } + *input = retained; + } + + if !eager_tools.is_empty() { + object.insert("tools".to_string(), Value::Array(eager_tools)); + } else { + object.remove("tools"); + } + if object + .get("tool_choice") + .and_then(Value::as_object) + .and_then(|choice| choice.get("type")) + .and_then(Value::as_str) + == Some("tool_search") + { + object.insert("tool_choice".to_string(), Value::String("auto".to_string())); + } +} + +fn is_tool_search(value: &Value) -> bool { + value.get("type").and_then(Value::as_str) == Some("tool_search") +} + +fn is_tool_search_replay(value: &Value) -> bool { + matches!( + value.get("type").and_then(Value::as_str), + Some("tool_search_call" | "tool_search_output") + ) +} + +// Flattens namespaces because providers without deferred discovery do not accept that wrapper. +fn push_eager_tool(mut tool: Value, output: &mut Vec) { + match tool.get("type").and_then(Value::as_str) { + Some("tool_search") => return, + Some("namespace") => { + if let Some(tools) = tool.get_mut("tools").and_then(Value::as_array_mut) { + for nested in std::mem::take(tools) { + push_eager_tool(nested, output); + } + } + return; + } + _ => {} + } + strip_defer_loading(&mut tool); + if !output.contains(&tool) { + output.push(tool); + } +} + +fn strip_defer_loading(value: &mut Value) { + match value { + Value::Object(object) => { + object.remove("defer_loading"); + for value in object.values_mut() { + strip_defer_loading(value); + } + } + Value::Array(values) => { + for value in values { + strip_defer_loading(value); + } + } + _ => {} + } +} + +/// Normalizes cross-provider Responses replay and hosted tools for xAI backends. +/// +/// Codex annotates `web_search` with OpenAI-only controls such as +/// `external_web_access` and `search_content_types`. xAI exposes the same hosted tool, +/// but rejects those fields. A false external-access flag removes the tool instead of +/// accidentally upgrading a disabled/cached request to live search. +pub(crate) fn normalize_xai_responses_request(body: &mut Value) { + let Some(object) = body.as_object_mut() else { + return; + }; + let Some(tools) = object.get_mut("tools").and_then(Value::as_array_mut) else { + return; + }; + + let mut normalized = Vec::with_capacity(tools.len()); + for tool in std::mem::take(tools) { + if !matches!( + tool.get("type").and_then(Value::as_str), + Some("web_search" | "web_search_preview") + ) { + normalized.push(tool); + continue; + } + if tool.get("external_web_access").and_then(Value::as_bool) == Some(false) { + continue; + } + + let mut search = + Map::from_iter([("type".to_string(), Value::String("web_search".to_string()))]); + if let Some(filters) = normalize_xai_search_filters(&tool) { + search.insert("filters".to_string(), filters); + } + copy_search_field( + &tool, + &mut search, + "enable_image_understanding", + "enable_image_understanding", + ); + copy_search_field( + &tool, + &mut search, + "enable_image_search", + "enable_image_search", + ); + normalized.push(Value::Object(search)); + } + + if normalized.is_empty() { + object.remove("tools"); + object.remove("tool_choice"); + } else { + *object.get_mut("tools").expect("tools exists") = Value::Array(normalized); + } +} + +fn normalize_xai_search_filters(tool: &Value) -> Option { + let source = tool.get("filters").unwrap_or(tool); + let mut filters = Map::new(); + copy_search_field(source, &mut filters, "allowed_domains", "allowed_domains"); + copy_search_field(source, &mut filters, "excluded_domains", "excluded_domains"); + copy_search_field(source, &mut filters, "blocked_domains", "excluded_domains"); + (!filters.is_empty()).then_some(Value::Object(filters)) +} + +/// Returns the first Responses web-search definition preserved on the inbound request. +pub(crate) fn responses_web_search_tool(request: &LlmRequest) -> Option { + let format = WireFormat::OpenAiResponses.into(); + request + .preservation + .requests + .get(&format) + .and_then(|body| body.get("tools")) + .and_then(Value::as_array) + .and_then(|tools| { + tools.iter().find(|tool| { + matches!( + tool.get("type").and_then(Value::as_str), + Some("web_search" | "web_search_preview") + ) + }) + }) + .cloned() +} + +/// Adds an Anthropic-native web-search definition translated from Responses. +pub(crate) fn bridge_responses_web_search_to_anthropic(body: &mut Value, source: &Value) { + let Some(body) = body.as_object_mut() else { + return; + }; + let tools = body + .entry("tools".to_string()) + .or_insert_with(|| Value::Array(Vec::new())); + let Some(tools) = tools.as_array_mut() else { + return; + }; + if tools.iter().any(|tool| { + tool.get("name").and_then(Value::as_str) == Some("web_search") + || tool + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| kind.starts_with("web_search_")) + }) { + return; + } + + let mut translated = Map::from_iter([ + ( + "type".to_string(), + Value::String("web_search_20250305".to_string()), + ), + ("name".to_string(), Value::String("web_search".to_string())), + ]); + copy_search_field(source, &mut translated, "max_uses", "max_uses"); + copy_search_field(source, &mut translated, "user_location", "user_location"); + copy_search_field( + source, + &mut translated, + "allowed_domains", + "allowed_domains", + ); + copy_search_field( + source, + &mut translated, + "blocked_domains", + "blocked_domains", + ); + if let Some(filters) = source.get("filters") { + copy_search_field( + filters, + &mut translated, + "allowed_domains", + "allowed_domains", + ); + copy_search_field( + filters, + &mut translated, + "blocked_domains", + "blocked_domains", + ); + copy_search_field( + filters, + &mut translated, + "excluded_domains", + "blocked_domains", + ); + } + tools.push(Value::Object(translated)); +} + +fn copy_search_field(source: &Value, target: &mut Map, from: &str, to: &str) { + if let Some(value) = source.get(from) { + target.insert(to.to_string(), value.clone()); + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + // Eager compatibility must retain callable and hosted tools while removing discovery records. + #[test] + fn eager_loading_flattens_tools_and_removes_search_replay() { + let mut body = json!({ + "tools": [ + { + "type": "namespace", + "name": "repo", + "tools": [{ + "type": "function", + "name": "read_file", + "defer_loading": true, + "parameters": {"type": "object"} + }] + }, + {"type": "mcp", "server_label": "docs", "defer_loading": true}, + {"type": "web_search"}, + {"type": "tool_search"} + ], + "tool_choice": {"type": "tool_search"}, + "input": [ + {"role": "user", "content": "Continue."}, + {"type": "tool_search_call", "call_id": "search_1"}, + { + "type": "tool_search_output", + "call_id": "search_1", + "tools": [{ + "type": "function", + "name": "write_file", + "defer_loading": true, + "parameters": {"type": "object"} + }] + }, + { + "type": "function_call", + "call_id": "call_1", + "name": "read_file", + "namespace": "repo", + "arguments": "{}" + } + ] + }); + + eager_load_responses_tool_search(&mut body); + + assert_eq!(body["tool_choice"], "auto"); + assert_eq!(body["tools"].as_array().map(Vec::len), Some(4)); + assert!(body["tools"].as_array().is_some_and(|tools| { + tools.iter().all(|tool| { + !matches!( + tool.get("type").and_then(Value::as_str), + Some("namespace" | "tool_search") + ) && tool.get("defer_loading").is_none() + }) + })); + assert!( + body["input"] + .as_array() + .is_some_and(|input| { input.iter().all(|item| !is_tool_search_replay(item)) }) + ); + assert_eq!(body["input"][1].get("namespace"), None); + } + + // Responses web search must become an Anthropic server tool, not a client function. + #[test] + fn web_search_maps_to_anthropic_server_tool() { + let mut body = json!({"messages": [{"role": "user", "content": "Search."}]}); + let source = json!({ + "type": "web_search", + "filters": {"allowed_domains": ["example.com"]}, + "user_location": {"type": "approximate", "country": "US"} + }); + + bridge_responses_web_search_to_anthropic(&mut body, &source); + + assert_eq!(body["tools"][0]["type"], "web_search_20250305"); + assert_eq!(body["tools"][0]["name"], "web_search"); + assert_eq!(body["tools"][0]["allowed_domains"], json!(["example.com"])); + assert_eq!(body["tools"][0]["user_location"]["country"], "US"); + } + + #[test] + fn xai_web_search_drops_openai_only_options_but_stays_live() { + let mut body = json!({ + "tools": [ + { + "type": "web_search", + "external_web_access": true, + "search_content_types": ["text", "image"], + "search_context_size": "high", + "user_location": {"type": "approximate", "country": "US"}, + "filters": {"allowed_domains": ["example.com"]} + }, + {"type": "function", "name": "echo", "parameters": {"type": "object"}} + ], + "tool_choice": "auto" + }); + + normalize_xai_responses_request(&mut body); + + assert_eq!( + body["tools"][0], + json!({ + "type": "web_search", + "filters": {"allowed_domains": ["example.com"]} + }) + ); + assert_eq!(body["tools"][1]["name"], "echo"); + assert_eq!(body["tool_choice"], "auto"); + } + + #[test] + fn xai_web_search_does_not_enable_disabled_external_access() { + let mut body = json!({ + "tools": [{"type": "web_search", "external_web_access": false}], + "tool_choice": "auto" + }); + + normalize_xai_responses_request(&mut body); + + assert_eq!(body.get("tools"), None); + assert_eq!(body.get("tool_choice"), None); + } +} diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 9ef133550..6a126a2a3 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -18,19 +18,23 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use parking_lot::Mutex; -use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive}; +use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive_with_decision_observer}; use switchyard_protocol::{ - Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason, + ContentBlock, Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient, + RoutingFallbackReason, completion_text, }; -use crate::observation::{LlmCallObservation, RunObservation, RunObserver}; +use crate::observation::{ + ClassifierContentObservation, LlmCallObservation, LlmCallStartObservation, RunObservation, + RunObserver, +}; use crate::{metrics, observability}; /// Run one request to completion, serving every offloaded model call with `client`. /// /// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published along -/// the way. `observer`, when present, receives each completed model call and, after a -/// successful routed run, its routing overhead. +/// the way. `observer`, when present, receives decisions and model-call lifecycle events +/// plus routing overhead after a successful routed run. /// /// `clients` resolves each offloaded call to the client for the target the algorithm /// selected — an algorithm may route among targets served by different providers, so this is @@ -44,24 +48,60 @@ pub async fn run( clients: ClientRouter, request: Request, observer: Option, +) -> Result<(Vec, Response)> { + run_with_observation_config( + algorithm, + clients, + request, + observer, + ObservationConfig::default(), + ) + .await +} + +/// Controls optional content captured for a [`RunObserver`]. +#[derive(Clone, Copy, Debug, Default)] +pub struct ObservationConfig { + /// Capture normalized prompts, reasoning, and verdicts for non-answer calls. + pub classifier_content: bool, +} + +/// Runs one request like [`run`] with explicit observation-content controls. +pub async fn run_with_observation_config( + algorithm: Arc, + clients: ClientRouter, + request: Request, + observer: Option, + observation_config: ObservationConfig, ) -> Result<(Vec, Response)> { let algorithm_name = algorithm.name().to_string(); // The output from `serve` goes in here: when each successful routed call was in // flight. Everything else the run spent time on is routing overhead. let routed_calls = Arc::new(Mutex::new(RoutedCallWindows::default())); let run_started = Instant::now(); - let result = drive(algorithm, request, { - let observer = observer.clone(); - let routed_calls = Arc::clone(&routed_calls); - move |call| { - serve( - clients.clone(), - call, - observer.clone(), - Arc::clone(&routed_calls), - ) - } - }) + let decision_observer = observer.clone(); + let result = drive_with_decision_observer( + algorithm, + request, + { + let observer = observer.clone(); + let routed_calls = Arc::clone(&routed_calls); + move |call| { + serve( + clients.clone(), + call, + observer.clone(), + observation_config, + Arc::clone(&routed_calls), + ) + } + }, + move |decision| { + if let Some(observer) = decision_observer.as_ref() { + observer(RunObservation::RoutingDecision(decision.clone())); + } + }, + ) .await?; if let Some(served) = routed_calls.lock().served() { let overhead = @@ -110,10 +150,18 @@ async fn serve( clients: ClientRouter, call: CallModel, observer: Option, + observation_config: ObservationConfig, // Output parameter because `drive` takes a function that returns a plain `Result<()>`. routed_calls: Arc>, ) -> Result<()> { - let result = call_first_available(&clients, &call, &observer, &routed_calls).await; + let result = call_first_available( + &clients, + &call, + &observer, + observation_config, + &routed_calls, + ) + .await; call.respond(result) } @@ -122,6 +170,7 @@ async fn call_first_available( clients: &ClientRouter, call: &CallModel, observer: &Option, + observation_config: ObservationConfig, routed_calls: &Arc>, ) -> Result { for (index, target) in call.models.iter().enumerate() { @@ -132,6 +181,7 @@ async fn call_first_available( request, call, observer, + observation_config, routed_calls, index, call.models.len(), @@ -200,6 +250,7 @@ async fn call_one( request: Request, call: &CallModel, observer: &Option, + observation_config: ObservationConfig, routed_calls: &Arc>, // index is for span log index: usize, @@ -217,9 +268,18 @@ async fn call_one( span.record("gen_ai.conversation.id", session_id); } let is_answer_call = call.is_answer_call; + let classifier_request = + (observer.is_some() && observation_config.classifier_content && !is_answer_call) + .then(|| request.llm_request.clone()); // Resolved before the clock starts: picking the client is Switchyard's work, not // the provider's, so it belongs in the routing overhead. let client = clients.route(model_id); + if let Some(observer) = observer { + observer(RunObservation::LlmCallStarted(LlmCallStartObservation { + selected_model: model_id.clone(), + is_answer_call, + })); + } let started = Instant::now(); let result = match client { Ok(client) => client.call(request).await, @@ -235,6 +295,22 @@ async fn call_one( }); let result = observability::observe_client_call(result); if let Some(observer) = observer { + if let Some(request) = classifier_request { + let aggregate = result + .as_ref() + .ok() + .and_then(|response| response.llm_response.as_agg()); + observer(RunObservation::ClassifierContent( + ClassifierContentObservation { + selected_model: model_id.clone(), + request, + reasoning: aggregate.and_then(classifier_reasoning), + verdict: aggregate.and_then(classifier_verdict), + is_success: result.is_ok(), + duration, + }, + )); + } observer(RunObservation::LlmCall(LlmCallObservation { selected_model: model_id.clone(), is_answer_call, @@ -254,6 +330,28 @@ async fn call_one( result } +fn classifier_reasoning(response: &switchyard_protocol::AggLlmResponse) -> Option { + nonempty_join(response.outputs.iter().flat_map(|output| { + output.content.iter().filter_map(|block| match block { + ContentBlock::Reasoning { text, .. } => Some(text.as_str()), + _ => None, + }) + })) +} + +fn classifier_verdict(response: &switchyard_protocol::AggLlmResponse) -> Option { + let verdict = completion_text(response); + (!verdict.is_empty()).then_some(verdict) +} + +fn nonempty_join<'a>(parts: impl Iterator) -> Option { + let joined = parts + .filter(|part| !part.is_empty()) + .collect::>() + .join("\n"); + (!joined.is_empty()).then_some(joined) +} + /// Whether a failed candidate is worth routing around. fn fallback_reason(error: &LibsyError) -> Option { let LibsyError::ClientCall { source, .. } = error else { @@ -574,6 +672,10 @@ mod tests { forward_auth: false, extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), + reasoning_effort_override: None, + bridge_custom_tools: false, + eager_load_tool_search: false, + xai_responses_compatibility: false, max_retries: 2, }) }; diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index dd5afa35e..24ffc644a 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -343,8 +343,18 @@ impl RoutedLlmClient for ClassifierClient { tokio::time::sleep(self.classifier_delay).await; r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"# }; + let mut response = text_response(Some(model_id.to_string()), completion); + if model_id == self.classifier_model_id { + response.outputs[0].content.insert( + 0, + ContentBlock::Reasoning { + text: "The task is bounded and supported.".to_string(), + signature: None, + }, + ); + } Ok(Response { - llm_response: LlmResponse::Agg(text_response(Some(model_id.to_string()), completion)), + llm_response: LlmResponse::Agg(response), metadata: None, }) } @@ -935,8 +945,17 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy:: Some(Some(MODEL)) ); let observations = observations.lock(); - assert_eq!(observations.len(), 2); - let RunObservation::LlmCall(observation) = &observations[0] else { + assert_eq!(observations.len(), 4); + let RunObservation::RoutingDecision(decision) = &observations[0] else { + return Err(test_error("expected a routing decision observation")); + }; + assert_eq!(decision.selected_model_id(), MODEL); + let RunObservation::LlmCallStarted(started) = &observations[1] else { + return Err(test_error("expected an LLM call start observation")); + }; + assert_eq!(started.selected_model, MODEL); + assert!(started.is_answer_call); + let RunObservation::LlmCall(observation) = &observations[2] else { return Err(test_error("expected an LLM call observation")); }; assert_eq!(observation.selected_model, MODEL); @@ -944,7 +963,7 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy:: assert!(observation.is_success); assert!(observation.usage.is_some()); assert!(matches!( - observations[1], + observations[3], RunObservation::RoutingOverhead(_) )); Ok(()) @@ -1283,6 +1302,57 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib Ok(()) } +#[tokio::test] +async fn observed_classifier_call_reports_prompt_reasoning_and_verdict() +-> switchyard_libsy::Result<()> { + let _guard = serialize_test().lock().await; + let observations = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&observations); + let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation)); + let client = Arc::new(ClassifierClient { + classifier_model_id: "classifier".into(), + classifier_delay: Duration::ZERO, + routed_delay: Duration::ZERO, + }) as Arc; + + switchyard_llm_client::run_with_observation_config( + classifier_router("classifier", "weak", "strong")?, + ClientRouter::single(client), + classifier_request(), + Some(observer), + switchyard_llm_client::ObservationConfig { + classifier_content: true, + }, + ) + .await?; + + let observations = observations.lock(); + let content = observations + .iter() + .find_map(|observation| match observation { + RunObservation::ClassifierContent(content) => Some(content), + _ => None, + }) + .ok_or_else(|| test_error("expected classifier content observation"))?; + assert_eq!(content.selected_model, "classifier"); + assert_eq!( + content.reasoning.as_deref(), + Some("The task is bounded and supported.") + ); + assert_eq!( + content.verdict.as_deref(), + Some( + r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"# + ) + ); + assert_eq!( + content.request.messages, + classifier_request().llm_request.messages + ); + assert!(content.is_success); + Ok(()) +} + #[tokio::test] async fn classifier_fail_open_records_each_failure_stage() -> switchyard_libsy::Result<()> { let _guard = serialize_test().lock().await; diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index be385a70e..89bccc992 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -13,11 +13,12 @@ //! private state value across turns with the same session ID. Requests without a session ID use //! unretained per-run state. //! -//! The selected target is offered first, followed by every other configured target. The consumer -//! may fall through that ordered candidate list when a model call fails. +//! By default, the selected target is offered first, followed by every other configured target. +//! The consumer may fall through that ordered candidate list when a model call fails. Callers may +//! disable target failover when a routing decision must remain authoritative. use std::{ - collections::HashMap, + collections::{BTreeSet, HashMap}, sync::{Arc, Once, Weak}, time::{Duration, Instant}, }; @@ -29,7 +30,8 @@ use tokio::sync::Mutex as AsyncMutex; use crate::core::algorithm::{self, Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::processor::{Event, Processor}; -use crate::{LibsyError, Result}; +use crate::target_modalities::eligible_targets; +use crate::{LibsyError, Result, TargetModalities}; use switchyard_protocol::{Decision, ModelId, Request, Response}; struct SessionState { @@ -95,6 +97,8 @@ pub struct FallThrough { processors: Vec>>, classifiers: Vec>>, targets: Vec, + target_modalities: Option, + target_failover: bool, session_states: Option>>, cleanup_started: Once, } @@ -108,6 +112,8 @@ impl FallThrough<()> { processors: Vec::new(), classifiers: Vec::new(), targets, + target_modalities: None, + target_failover: true, session_states: None, cleanup_started: Once::new(), } @@ -126,6 +132,8 @@ where processors: Vec::new(), classifiers: Vec::new(), targets, + target_modalities: None, + target_failover: true, session_states: Some(Arc::new(Mutex::new(HashMap::new()))), cleanup_started: Once::new(), } @@ -154,6 +162,19 @@ where self.classifiers.push(classifier); self } + + /// Restricts routing and request-local fallback to modality-compatible targets. + pub fn with_target_modalities(mut self, target_modalities: TargetModalities) -> Self { + self.target_modalities = Some(target_modalities); + self + } + + /// Controls whether a failed selected target may fall through to another route target. + pub fn with_target_failover(mut self, enabled: bool) -> Self { + self.target_failover = enabled; + self + } + /// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint. pub(crate) async fn execute(&self, driver: Driver, request: Request) -> Result { self.start_cleanup_task(); @@ -186,7 +207,7 @@ where // it, later components see the rewrite, and the final value reaches the model. let mut request = request; let session_state = self.session_state(&request); - let (target, served) = match session_state { + let (target, served, eligible) = match session_state { Some(state) => { let mut state = state.lock().await; self.route(&mut state, &driver, &mut request).await? @@ -206,7 +227,7 @@ where Some(response) => Ok(response), None => { driver - .call_model(request, self.candidates(&target), true) + .call_model(request, self.candidates(&target, eligible.as_deref()), true) .await } } @@ -220,10 +241,14 @@ where } /// The selected target first, then every other configured target as a fallback candidate. - fn candidates(&self, target: &ModelId) -> Vec { + fn candidates(&self, target: &ModelId, eligible: Option<&[ModelId]>) -> Vec { + if !self.target_failover { + return vec![target.clone()]; + } + let candidates = eligible.unwrap_or(&self.targets); std::iter::once(target.clone()) .chain( - self.targets + candidates .iter() .filter(|candidate| *candidate != target) .cloned(), @@ -250,17 +275,63 @@ where state: &mut S, driver: &Driver, request: &mut Request, - ) -> Result<(ModelId, Option)> { + ) -> Result<(ModelId, Option, Option>)> { // 1. Processor chain accumulates request-side facts into the composition's state. for processor in &self.processors { processor.process(state, Event::Request(request)).await?; } + // Eligibility is computed after request processors so classifiers and + // fallback candidates see the final request-side content. + let eligibility = self + .target_modalities + .as_ref() + .map(|modalities| eligible_targets(&self.targets, modalities, request)) + .transpose()?; + let eligible_set = eligibility + .as_ref() + .map(|(_, eligible)| eligible.iter().cloned().collect::>()); + if let Some((required, eligible)) = &eligibility { + tracing::info!( + required_modalities = ?required, + eligible_targets = ?eligible, + "computed modality-compatible targets" + ); + let needs_scoring = self + .classifiers + .iter() + .any(|classifier| classifier.needs_single_eligible_scoring()); + if let [target] = eligible.as_slice() + && !needs_scoring + { + tracing::info!( + target = %target, + required_modalities = ?required, + "input modalities forced target selection" + ); + let target = target.clone(); + self.publish_decision(state, driver, request, &target) + .await?; + return Ok((target, None, Some(eligible.clone()))); + } + } + // 2. Fall through the cascade: the first classifier to score decides (argmax). The // per-request driver is offered to each — driver-backed classifiers use it. let mut routed = None; for classifier in &self.classifiers { - let (scores, response) = classifier.score(state, request, Some(driver)).await?; + let (scores, response) = match eligible_set.as_ref() { + Some(eligible) => { + classifier + .score_with_eligible_targets(state, request, Some(driver), eligible) + .await? + } + None => classifier.score(state, request, Some(driver)).await?, + }; + let scores = match eligible_set.as_ref() { + Some(eligible) => scores.retain_eligible(eligible), + None => scores, + }; if let Some(score) = scores.argmax(false)? { // Only the deciding classifier's response answers the turn; an abstaining // classifier selected nothing for it to be the answer to. @@ -268,10 +339,26 @@ where break; } } - let Some((score, deciding, served)) = routed else { - return Err(LibsyError::AlgorithmError { - message: "every classifier abstained".to_string(), - }); + let (score, deciding, served) = match routed { + Some(routed) => routed, + None => { + let Some((_, eligible)) = &eligibility else { + return Err(LibsyError::AlgorithmError { + message: "every classifier abstained".to_string(), + }); + }; + let Some(target) = eligible.first() else { + return Err(LibsyError::NoTargets); + }; + tracing::info!( + target = %target, + "classifier cascade fell back to first compatible target" + ); + let target = target.clone(); + self.publish_decision(state, driver, request, &target) + .await?; + return Ok((target, None, Some(eligible.clone()))); + } }; // 3. Resolve the target, log the choice, and publish the decision. @@ -280,10 +367,24 @@ where let message = (self.decision_reason)(&self.name, &score); let message = with_routing_tier(message, deciding.routing_tier(&target)); tracing::info!("{message}"); + self.publish_decision(state, driver, request, &target) + .await?; + + Ok((target, served, eligibility.map(|(_, eligible)| eligible))) + } + + /// Publishes a choice and replays it through post-decision processors. + async fn publish_decision( + &self, + state: &mut S, + driver: &Driver, + request: &mut Request, + target: &ModelId, + ) -> Result<()> { let decision: Decision = Decision::new(target.clone(), true); driver.decide(decision.clone()).await?; - // 4. Post-decision replay: every processor sees the decision so stateful ones + // Post-decision replay: every processor sees the decision so stateful ones // can bind it, and may rewrite the outbound request (e.g. add a target prompt). for processor in &self.processors { let event = Event::Decision { @@ -292,8 +393,7 @@ where }; processor.process(state, event).await?; } - - Ok((target, served)) + Ok(()) } } @@ -362,13 +462,18 @@ where } #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use super::*; use crate::algorithms::util::prompts; use crate::core::classifier::Classification; use crate::{SystemPromptProcessor, TargetPrompts}; use crate::core::testing::{Serve, echo, reply, test_drive}; - use switchyard_protocol::{LlmRequest, Message, Metadata, Role, completion_text, text_request}; + use switchyard_protocol::{ + ContentBlock, ImageSource, InputModality, LlmRequest, Message, Metadata, Role, + completion_text, text_request, + }; #[derive(Debug, thiserror::Error)] #[error("{0}")] @@ -446,6 +551,40 @@ mod tests { names.iter().map(|name| ModelId::from(*name)).collect() } + fn target_modalities(entries: &[(&str, &[InputModality])]) -> TargetModalities { + entries + .iter() + .map(|(target, modalities)| { + (ModelId::from(*target), modalities.iter().copied().collect()) + }) + .collect() + } + + fn image_request() -> Request { + Request { + llm_request: LlmRequest { + model: Some("auto".to_string()), + messages: vec![Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "describe this".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/image.png".to_string(), + detail: None, + }, + }, + ], + }], + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + fn target_prompts() -> TargetPrompts { TargetPrompts::default() .with("capable", CAPABLE_PROMPT) @@ -676,6 +815,27 @@ mod tests { Err(test_error("expected a CallModel step")) } + #[tokio::test] + async fn target_failover_can_be_disabled() -> Result<()> { + use futures::StreamExt; + + let router = Arc::new( + FallThrough::<()>::new(target_set(&["weak", "mid", "strong"])) + .with_target_failover(false) + .with_classifier(fixed(vec![score("mid", 0.9)])), + ); + let stream = router.run_stream(request()); + tokio::pin!(stream); + while let Some(step) = stream.next().await { + if let crate::Step::CallModel(call) = step? { + assert_eq!(call.models, target_set(&["mid"])); + assert_eq!(call.request.llm_request.model.as_deref(), Some("mid")); + return Ok(()); + } + } + Err(test_error("expected a CallModel step")) + } + #[tokio::test] async fn argmax_picks_the_highest_confidence_target() -> Result<()> { let router = FallThrough::<()>::new(target_set(&["strong", "weak"])) @@ -963,6 +1123,115 @@ mod tests { assert!(!states.contains_key("session-1")); } + #[tokio::test] + async fn modalities_force_the_only_compatible_target_without_stripping_content() -> Result<()> { + let captured = Arc::new(Mutex::new(None)); + let original = image_request(); + let router = Arc::new( + FallThrough::new(target_set(&["text", "vision"])) + .with_target_modalities(target_modalities(&[ + ("text", &[InputModality::Text]), + ("vision", &[InputModality::Text, InputModality::Image]), + ])) + // This incompatible choice must never run because eligibility is decisive. + .with_classifier(fixed(vec![score("text", 1.0)])), + ); + + let (selected, trace) = + run_request(&router, original.clone(), capturing(captured.clone())).await?; + + assert_eq!(selected, "vision"); + assert_eq!(trace[0].selected_model_id(), "vision"); + let sent = captured.lock().clone().expect("vision target was called"); + assert_eq!( + sent.llm_request.messages[0].content, + original.llm_request.messages[0].content + ); + Ok(()) + } + + #[tokio::test] + async fn incompatible_classifier_scores_are_ignored_during_the_cascade() -> Result<()> { + let router = Arc::new( + FallThrough::new(target_set(&["vision-first", "vision-second", "text"])) + .with_target_modalities(target_modalities(&[ + ("vision-first", &[InputModality::Text, InputModality::Image]), + ( + "vision-second", + &[InputModality::Text, InputModality::Image], + ), + ("text", &[InputModality::Text]), + ])) + .with_classifier(fixed(vec![score("text", 1.0)])) + .with_classifier(fixed(vec![score("vision-second", 0.5)])), + ); + + let (selected, _) = run_request(&router, image_request(), echo()).await?; + + assert_eq!(selected, "vision-second"); + Ok(()) + } + + #[tokio::test] + async fn incompatible_default_falls_back_to_first_compatible_route_target() -> Result<()> { + let router = Arc::new( + FallThrough::new(target_set(&["vision-first", "vision-second", "text"])) + .with_target_modalities(target_modalities(&[ + ("vision-first", &[InputModality::Text, InputModality::Image]), + ( + "vision-second", + &[InputModality::Text, InputModality::Image], + ), + ("text", &[InputModality::Text]), + ])) + .with_classifier(Arc::new(DefaultTarget::new("text"))), + ); + + let (selected, _) = run_request(&router, image_request(), echo()).await?; + + assert_eq!(selected, "vision-first"); + Ok(()) + } + + #[tokio::test] + async fn no_compatible_target_is_typed_and_emits_no_model_call() { + let calls = Arc::new(AtomicUsize::new(0)); + let called = Arc::clone(&calls); + let router: Arc = Arc::new( + FallThrough::new(target_set(&["text-a", "text-b"])) + .with_target_modalities(target_modalities(&[ + ("text-a", &[InputModality::Text]), + ("text-b", &[InputModality::Text]), + ])) + .with_classifier(Arc::new(DefaultTarget::new("text-a"))), + ); + + let result = test_drive(router, image_request(), move |target: ModelId, _request| { + let called = Arc::clone(&called); + async move { + called.fetch_add(1, Ordering::Relaxed); + Ok(reply(target)) + } + }) + .await; + + match result { + Err(LibsyError::NoCompatibleTargets { + required_modalities, + target_modalities, + }) => { + assert_eq!( + required_modalities, + BTreeSet::from([InputModality::Text, InputModality::Image]) + ); + assert_eq!(target_modalities.len(), 2); + } + Err(error) => panic!("expected NoCompatibleTargets, got {error:?}"), + Ok(_) => panic!("expected NoCompatibleTargets, routing succeeded"), + } + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + #[test] fn cleanup_removes_only_inactive_idle_sessions() { let router = FallThrough::::new_with_state(target_set(&["strong"])); diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 7d8b2fb66..42d7c9994 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -13,7 +13,7 @@ use switchyard_protocol::{ContentBlock, Message, ModelId, Role}; use super::fall_through::{DefaultTarget, FallThrough}; use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; -use super::util::affinity::AffinityRouter; +use super::util::affinity::{AffinityRouter, TurnAffinityRouter, is_user_turn_message}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, }; @@ -26,7 +26,7 @@ use super::util::target_selector::TargetSelectorPolicy; use crate::core::algorithm::{self, Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::{State, StateValue}; -use crate::{LibsyError, Result}; +use crate::{LibsyError, Result, TargetModalities}; use switchyard_protocol::{AggLlmResponse, LlmClientError, LlmResponse, Request, Response}; const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md"); @@ -136,9 +136,11 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { counted } -/// Keeps the opening task and the latest user follow-up when they differ. +/// Keeps the opening task and latest human user follow-up, excluding tool-loop state. fn task_messages(messages: &[Message]) -> Vec { - let mut user_messages = messages.iter().filter(|message| message.role == Role::User); + let mut user_messages = messages + .iter() + .filter(|message| is_user_turn_message(message)); let Some(opening_task) = user_messages.next() else { return Vec::new(); }; @@ -148,19 +150,52 @@ fn task_messages(messages: &[Message]) -> Vec { } } +fn text_only_classifier_content(content: Vec) -> Vec { + content + .into_iter() + .map(|block| match block { + ContentBlock::Image { .. } => ContentBlock::Text { + text: "[image input omitted from text-only classifier]".to_string(), + }, + ContentBlock::Audio { .. } => ContentBlock::Text { + text: "[audio input omitted from text-only classifier]".to_string(), + }, + ContentBlock::Video { .. } => ContentBlock::Text { + text: "[video input omitted from text-only classifier]".to_string(), + }, + ContentBlock::File { .. } => ContentBlock::Text { + text: "[file input omitted from text-only classifier]".to_string(), + }, + ContentBlock::ToolResult(mut result) => { + result.content = text_only_classifier_content(result.content); + ContentBlock::ToolResult(result) + } + block => block, + }) + .collect() +} + /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, + text_only: bool, } impl ClassifierInput for TaskInput { fn build_messages(&self, _state: &State, request: &Request) -> Vec { // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. - match self.recent_turn_window { + let mut messages = match self.recent_turn_window { Some(window) => trim_messages(&request.llm_request.messages, window), None => task_messages(&request.llm_request.messages), + }; + if self.text_only { + for message in &mut messages { + message.content = + text_only_classifier_content(std::mem::take(&mut message.content)); + } } + messages } } @@ -233,6 +268,8 @@ pub struct TaskClassifierConfig { pub threshold_step: f64, /// Enables session affinity before the judge-backed classifier. pub session_affinity: bool, + /// Retains the selected target for tool-loop continuations within one user turn. + pub turn_affinity: bool, /// Uses the first user message as the SessionKey for sticky routing when session metadata is unavailable. pub message_hash_fallback: bool, /// Trailing conversation turns the judge sees on top of the client @@ -258,6 +295,8 @@ struct TaskClassifierConfigWire { #[serde(default)] session_affinity: bool, #[serde(default)] + turn_affinity: bool, + #[serde(default)] message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, @@ -284,6 +323,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, session_affinity: wire.session_affinity, + turn_affinity: wire.turn_affinity, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, contract, @@ -302,6 +342,7 @@ impl Default for TaskClassifierConfig { base_threshold: 0.0, threshold_step: 0.0, session_affinity: false, + turn_affinity: false, message_hash_fallback: false, recent_turn_window: None, contract: ClassifierContractConfig::default(), @@ -347,6 +388,11 @@ impl TaskClassifierConfig { message: "message_hash_fallback requires session_affinity".to_string(), }); } + if self.session_affinity && self.turn_affinity { + return Err(LibsyError::AlgorithmError { + message: "session_affinity and turn_affinity cannot both be enabled".to_string(), + }); + } Ok(()) } } @@ -381,10 +427,14 @@ pub struct CustomClassifierConfig { pub policy: CustomClassifierPolicy, /// Enables session affinity before the judge-backed classifier. pub session_affinity: bool, + /// Retains the selected target for tool-loop continuations within one user turn. + pub turn_affinity: bool, /// Uses the first user message when session metadata is unavailable. pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, + /// Replaces media payloads with bounded text placeholders before judging. + pub judge_text_only: bool, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, } @@ -401,8 +451,10 @@ impl CustomClassifierConfig { response_schema, policy, session_affinity: false, + turn_affinity: false, message_hash_fallback: false, recent_turn_window: None, + judge_text_only: false, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } } @@ -418,6 +470,11 @@ impl CustomClassifierConfig { message: "message_hash_fallback requires session_affinity".to_string(), }); } + if self.session_affinity && self.turn_affinity { + return Err(LibsyError::AlgorithmError { + message: "session_affinity and turn_affinity cannot both be enabled".to_string(), + }); + } Ok(()) } } @@ -581,6 +638,28 @@ impl Classifier for EscalationClassifier { Ok((decisive(&self.efficient), Some(efficient_response))) } + + async fn score_with_eligible_targets( + &self, + state: &mut State, + request: &mut Request, + driver: Option<&Driver>, + eligible_targets: &BTreeSet, + ) -> Result<(Classification, Option)> { + // The efficient pre-call is part of classification, so it must not run + // when that tier cannot accept the request. + if !eligible_targets.contains(&self.efficient) { + return Ok(( + if eligible_targets.contains(&self.capable) { + decisive(&self.capable) + } else { + Classification::Ambiguous(Vec::new()) + }, + None, + )); + } + self.score(state, request, driver).await + } } /// Routes requests through a capability, escalation, or custom classifier mode. @@ -593,6 +672,7 @@ pub struct LlmTaskClassifier { struct ClassifierRouteConfig { default_target: ModelId, session_affinity: bool, + turn_affinity: bool, message_hash_fallback: bool, } @@ -677,6 +757,18 @@ impl LlmTaskClassifier { } } + /// Restricts classifier decisions and fallback calls to compatible targets. + pub fn with_target_modalities(mut self, target_modalities: TargetModalities) -> Self { + self.route = self.route.with_target_modalities(target_modalities); + self + } + + /// Controls whether a failed selected target may fall through to another classifier target. + pub fn with_target_failover(mut self, enabled: bool) -> Self { + self.route = self.route.with_target_failover(enabled); + self + } + fn build_capability( judge_target: ModelId, efficient_target: ModelId, @@ -687,12 +779,14 @@ impl LlmTaskClassifier { let contract = Self::load_capability_contract(&config.contract)?; let targets = vec![efficient_target.clone(), capable_target.clone()]; let session_affinity = config.session_affinity; + let turn_affinity = config.turn_affinity; let message_hash_fallback = config.message_hash_fallback; let classifier = Arc::new(TaskClassifier { classifier: JudgeClassifier::new( StructuredJudge::new( TaskInput { recent_turn_window: config.recent_turn_window, + text_only: false, }, contract, SerdeDecoder::new(), @@ -715,6 +809,7 @@ impl LlmTaskClassifier { ClassifierRouteConfig { default_target: classifier.capable_target.clone(), session_affinity, + turn_affinity, message_hash_fallback, }, ) @@ -772,8 +867,10 @@ impl LlmTaskClassifier { response_schema, policy, session_affinity, + turn_affinity, message_hash_fallback, recent_turn_window, + judge_text_only, max_output_tokens, } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; @@ -786,7 +883,10 @@ impl LlmTaskClassifier { }; let classifier: Arc> = Arc::new(JudgeClassifier::new( StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + text_only: judge_text_only, + }, contract, JsonSchemaDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, @@ -801,6 +901,7 @@ impl LlmTaskClassifier { ClassifierRouteConfig { default_target: default_name, session_affinity, + turn_affinity, message_hash_fallback, }, ) @@ -857,6 +958,11 @@ impl LlmTaskClassifier { message: "message_hash_fallback requires session_affinity".to_string(), }); } + if config.session_affinity && config.turn_affinity { + return Err(LibsyError::AlgorithmError { + message: "session_affinity and turn_affinity cannot both be enabled".to_string(), + }); + } // Affinity comes first so a retained assignment short-circuits the judge call. // Note: when this classifier is embedded inside another cascade (e.g. StageRouter) // the affinity processor never fires — only the inner score() is called. @@ -873,6 +979,12 @@ impl LlmTaskClassifier { .with_processor(affinity.clone()) .with_classifier(affinity); } + if config.turn_affinity { + let affinity = Arc::new(TurnAffinityRouter::new()); + route = route + .with_processor(affinity.clone()) + .with_classifier(affinity); + } let fallback = DefaultTarget::new(config.default_target); Ok(Self { route: route @@ -921,6 +1033,18 @@ impl Classifier for LlmTaskClassifier { ) -> Result<(Classification, Option)> { self.inner.score(state, request, driver).await } + + async fn score_with_eligible_targets( + &self, + state: &mut State, + request: &mut Request, + driver: Option<&Driver>, + eligible_targets: &BTreeSet, + ) -> Result<(Classification, Option)> { + self.inner + .score_with_eligible_targets(state, request, driver, eligible_targets) + .await + } } #[async_trait] @@ -943,8 +1067,9 @@ mod tests { use super::*; use switchyard_protocol::{ - ContentBlock, InstructionBlock, LlmClientError, LlmRequest, LlmResponseChunk, Metadata, - ToolCall, ToolResult, completion_text, text_request, text_response, + ContentBlock, ImageSource, InputModality, InstructionBlock, LlmClientError, LlmRequest, + LlmResponseChunk, Metadata, ToolCall, ToolResult, completion_text, text_request, + text_response, }; use crate::algorithms::util::llm_judge::Judge; @@ -1223,6 +1348,92 @@ mod tests { Ok(()) } + #[tokio::test] + async fn turn_affinity_reuses_a_target_until_the_next_user_message() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + config: TaskClassifierConfig { + turn_affinity: true, + ..test_config(TEST_THRESHOLD) + }, + })?); + + test_drive(router.clone(), classify_session_request(), recorder.serve()).await?; + + let mut continuation = classify_session_request(); + continuation.llm_request.messages.push(tool_call("call-1")); + continuation.llm_request.messages.push(Message { + role: Role::User, + content: tool_result("call-1").content, + }); + test_drive(router.clone(), continuation, recorder.serve()).await?; + + let mut follow_up = classify_follow_up_request(); + follow_up.metadata = classify_session_request().metadata; + test_drive(router, follow_up, recorder.serve()).await?; + + assert_eq!( + recorder.calls(), + vec!["judge", "efficient", "efficient", "judge", "efficient"] + ); + Ok(()) + } + + #[tokio::test] + async fn turn_affinity_rebinds_when_modalities_make_the_target_ineligible() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let modalities = BTreeMap::from([ + ( + ModelId::from("efficient"), + BTreeSet::from([InputModality::Text]), + ), + ( + ModelId::from("capable"), + BTreeSet::from([InputModality::Text, InputModality::Image]), + ), + ]); + let router = Arc::new( + LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + config: TaskClassifierConfig { + turn_affinity: true, + ..test_config(TEST_THRESHOLD) + }, + })? + .with_target_modalities(modalities), + ); + + test_drive(router.clone(), classify_session_request(), recorder.serve()).await?; + + let mut image_continuation = classify_session_request(); + image_continuation.llm_request.messages.push(Message { + role: Role::User, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "view-image".to_string(), + content: vec![ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/image.png".to_string(), + detail: None, + }, + }], + is_error: None, + })], + }); + test_drive(router.clone(), image_continuation, recorder.serve()).await?; + test_drive(router, classify_session_request(), recorder.serve()).await?; + + assert_eq!( + recorder.calls(), + vec!["judge", "efficient", "capable", "capable"] + ); + Ok(()) + } + #[tokio::test] async fn classifier_config_reuses_message_hash_affinity_for_a_follow_up() -> Result<()> { let recorder = Arc::new(Recorder::default()); @@ -1411,7 +1622,10 @@ mod tests { /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { Ok(StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + text_only: false, + }, LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, @@ -1465,6 +1679,23 @@ mod tests { Ok(()) } + #[test] + fn default_task_context_excludes_user_role_tool_results() { + let messages = vec![ + Message::text(Role::User, "initial task"), + tool_call("call-1"), + Message { + role: Role::User, + content: tool_result("call-1").content, + }, + ]; + + assert_eq!( + task_messages(&messages), + vec![Message::text(Role::User, "initial task")] + ); + } + fn tool_call(id: &str) -> Message { Message { role: Role::Assistant, @@ -1582,6 +1813,81 @@ mod tests { ); } + #[test] + fn text_only_judge_replaces_direct_and_tool_result_images() { + let request = Request { + llm_request: LlmRequest { + messages: vec![ + Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "Inspect this screenshot.".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/private.png".to_string(), + detail: None, + }, + }, + ], + }, + tool_call("view-image"), + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "view-image".to_string(), + content: vec![ + ContentBlock::Text { + text: "Local render".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Base64 { + media_type: Some("image/png".to_string()), + data: "private-image-bytes".to_string(), + }, + }, + ], + is_error: None, + })], + }, + ], + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + + let messages = TaskInput { + recent_turn_window: Some(8), + text_only: true, + } + .build_messages(&State::default(), &request); + + assert!( + request + .llm_request + .input_modalities() + .contains(&InputModality::Image), + "the completion request must retain its original image" + ); + assert_eq!( + messages[0].content[1], + ContentBlock::Text { + text: "[image input omitted from text-only classifier]".to_string(), + } + ); + let ContentBlock::ToolResult(result) = &messages[2].content[0] else { + panic!("expected the tool result to remain structured"); + }; + assert_eq!( + result.content[1], + ContentBlock::Text { + text: "[image input omitted from text-only classifier]".to_string(), + } + ); + } + #[test] fn capability_judge_builds_a_structured_request() -> Result<()> { let judge = capability_judge(None)?; @@ -1682,6 +1988,7 @@ mod tests { let judge: CapabilityJudge = StructuredJudge::new( TaskInput { recent_turn_window: None, + text_only: false, }, contract, SerdeDecoder::new(), diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs index 3de5ccc9a..c74f92266 100644 --- a/crates/libsy/src/algorithms/passthrough.rs +++ b/crates/libsy/src/algorithms/passthrough.rs @@ -7,13 +7,15 @@ use std::sync::Arc; use switchyard_protocol::{ModelId, Request, Response}; -use crate::Result; use crate::core::algorithm::{Algorithm, Driver}; +use crate::target_modalities::eligible_targets; +use crate::{Result, TargetModalities}; use switchyard_protocol::Decision; /// Routing algorithm that always calls one configured target. pub struct Passthrough { target: ModelId, + target_modalities: Option, } impl Passthrough { @@ -21,8 +23,15 @@ impl Passthrough { pub fn new(target: impl Into) -> Self { Passthrough { target: target.into(), + target_modalities: None, } } + + /// Validates requests against the sole target's declared input modalities. + pub fn with_target_modalities(mut self, target_modalities: TargetModalities) -> Self { + self.target_modalities = Some(target_modalities); + self + } } #[async_trait::async_trait] @@ -32,6 +41,18 @@ impl Algorithm for Passthrough { } async fn route(self: Arc, driver: Driver, request: Request) -> Result { + if let Some(target_modalities) = &self.target_modalities { + let (required, eligible) = eligible_targets( + std::slice::from_ref(&self.target), + target_modalities, + &request, + )?; + tracing::info!( + required_modalities = ?required, + eligible_targets = ?eligible, + "computed modality-compatible targets" + ); + } tracing::info!(target = %self.target, "passthrough selected target"); let decision: Decision = Decision::new(self.target.clone(), true); driver.decide(decision.clone()).await?; @@ -43,12 +64,17 @@ impl Algorithm for Passthrough { #[cfg(test)] mod tests { + use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use super::Passthrough; + use crate::LibsyError; use crate::core::algorithm::Algorithm; use crate::core::testing::{echo, test_drive}; - use switchyard_protocol::{Request, completion_text, text_request}; + use switchyard_protocol::{ + ContentBlock, ImageSource, InputModality, LlmRequest, Message, Request, Role, + completion_text, text_request, + }; #[tokio::test] async fn test_passthrough() -> crate::Result<()> { @@ -74,4 +100,38 @@ mod tests { assert!(trace[0].is_answer_call()); Ok(()) } + + #[tokio::test] + async fn passthrough_rejects_an_incompatible_request_before_calling() { + const MODEL_ID: &str = "testing/text-only"; + let algorithm: Arc = Arc::new( + Passthrough::new(MODEL_ID).with_target_modalities(BTreeMap::from([( + MODEL_ID.into(), + BTreeSet::from([InputModality::Text]), + )])), + ); + let request = Request { + llm_request: LlmRequest { + messages: vec![Message { + role: Role::User, + content: vec![ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/image.png".to_string(), + detail: None, + }, + }], + }], + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + + let result = test_drive(algorithm, request, echo()).await; + + assert!(matches!( + result, + Err(LibsyError::NoCompatibleTargets { .. }) + )); + } } diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs index 4be406f32..20bb8d4e1 100644 --- a/crates/libsy/src/algorithms/rand.rs +++ b/crates/libsy/src/algorithms/rand.rs @@ -18,12 +18,13 @@ use rand::rngs::StdRng; use crate::algorithms::fall_through::FallThrough; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; -use crate::{LibsyError, Result}; +use crate::{LibsyError, Result, TargetModalities}; use switchyard_protocol::{ModelId, Request, Response}; /// Stateless weighted classifier used by random fall-through routing. pub struct RandomClassifier { targets: Vec, + weights: Vec, distribution: WeightedIndex, rng: Mutex, } @@ -76,14 +77,15 @@ impl RandomClassifier { "at least one weight must be positive".to_string(), )); } - let distribution = - WeightedIndex::new(weights).map_err(|error| invalid_weights(error.to_string()))?; + let distribution = WeightedIndex::new(weights.clone()) + .map_err(|error| invalid_weights(error.to_string()))?; let rng = match seed { Some(seed) => StdRng::seed_from_u64(seed), None => rand::make_rng(), }; Ok(Self { targets, + weights, distribution, rng: Mutex::new(rng), }) @@ -94,6 +96,30 @@ impl RandomClassifier { let index = self.distribution.sample(&mut *rng); self.targets[index].clone() } + + /// Samples from compatible targets using their original relative weights. + fn select_eligible_target(&self, eligible_targets: &BTreeSet) -> Result { + let eligible = self + .targets + .iter() + .zip(&self.weights) + .filter(|(target, _)| eligible_targets.contains(*target)) + .collect::>(); + let weights = eligible + .iter() + .map(|(_, weight)| **weight) + .collect::>(); + if !weights.iter().any(|weight| *weight > 0.0) { + return Err(LibsyError::AlgorithmError { + message: "no modality-compatible random target has a positive weight".to_string(), + }); + } + let distribution = + WeightedIndex::new(weights).map_err(|error| invalid_weights(error.to_string()))?; + let mut rng = self.rng.lock(); + let index = distribution.sample(&mut *rng); + Ok(eligible[index].0.clone()) + } } fn invalid_weights(message: String) -> LibsyError { @@ -107,6 +133,10 @@ impl Classifier for RandomClassifier where S: Send + 'static, { + fn needs_single_eligible_scoring(&self) -> bool { + true + } + async fn score( &self, _state: &mut S, @@ -121,6 +151,22 @@ where None, )) } + + async fn score_with_eligible_targets( + &self, + _state: &mut S, + _request: &mut Request, + _driver: Option<&Driver>, + eligible_targets: &BTreeSet, + ) -> Result<(Classification, Option)> { + Ok(( + Classification::Scores(vec![Score { + confidence: 1.0, + target: self.select_eligible_target(eligible_targets)?, + }]), + None, + )) + } } /// Random router implemented as a stateless fall-through composition. @@ -146,6 +192,12 @@ impl Random { .with_classifier(classifier); Ok(Self { inner }) } + + /// Restricts weighted selection and fallback to modality-compatible targets. + pub fn with_target_modalities(mut self, target_modalities: TargetModalities) -> Self { + self.inner = self.inner.with_target_modalities(target_modalities); + self + } } fn random_decision_reason(_name: &str, winner: &Score) -> String { @@ -168,7 +220,10 @@ mod tests { use super::*; use std::collections::HashSet; - use switchyard_protocol::{Metadata, completion_text, text_request}; + use switchyard_protocol::{ + ContentBlock, ImageSource, InputModality, LlmRequest, Message, Metadata, Role, + completion_text, text_request, + }; use crate::algorithms::util::affinity::AffinityRouter; use crate::core::testing::{echo, test_drive}; @@ -196,6 +251,40 @@ mod tests { names.iter().map(|name| ModelId::from(*name)).collect() } + fn target_modalities(entries: &[(&str, &[InputModality])]) -> TargetModalities { + entries + .iter() + .map(|(target, modalities)| { + (ModelId::from(*target), modalities.iter().copied().collect()) + }) + .collect() + } + + fn image_request() -> Request { + Request { + llm_request: LlmRequest { + model: Some("auto".to_string()), + messages: vec![Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "describe this".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/image.png".to_string(), + detail: None, + }, + }, + ], + }], + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + fn algorithm(names: &[&str], weights: Option>, seed: Option) -> Result { Random::new(target_set(names), weights, seed) } @@ -311,6 +400,76 @@ mod tests { Ok(()) } + #[tokio::test] + async fn weighted_selection_is_restricted_and_renormalized_over_compatible_targets() + -> Result<()> { + fn modality_aware_random() -> Result> { + let router = Random::new( + target_set(&["text", "vision-a", "vision-b"]), + Some(vec![100.0, 1.0, 3.0]), + Some(42), + )? + .with_target_modalities(target_modalities(&[ + ("text", &[InputModality::Text]), + ("vision-a", &[InputModality::Text, InputModality::Image]), + ("vision-b", &[InputModality::Text, InputModality::Image]), + ])); + Ok(Arc::new(router)) + } + + async fn selections(router: Arc) -> Result> { + let mut selected = Vec::new(); + for _ in 0..1_000 { + let (_, response) = test_drive(router.clone(), image_request(), echo()).await?; + selected.push( + response + .llm_response + .as_agg() + .map(completion_text) + .unwrap_or_default(), + ); + } + Ok(selected) + } + + let first = selections(modality_aware_random()?).await?; + let second = selections(modality_aware_random()?).await?; + + assert_eq!(first, second); + assert!(first.iter().all(|target| target != "text")); + let vision_b = first + .iter() + .filter(|target| target.as_str() == "vision-b") + .count(); + assert!( + (700..=800).contains(&vision_b), + "expected a roughly 25/75 split, selected vision-b {vision_b} times" + ); + Ok(()) + } + + #[tokio::test] + async fn rejects_a_zero_weight_only_compatible_target() -> Result<()> { + // Modality filtering must not re-enable a target disabled by a zero weight. + let router: Arc = Arc::new( + algorithm(&["text", "vision"], Some(vec![1.0, 0.0]), Some(42))?.with_target_modalities( + target_modalities(&[ + ("text", &[InputModality::Text]), + ("vision", &[InputModality::Text, InputModality::Image]), + ]), + ), + ); + + let error = test_drive(router, image_request(), echo()).await.err(); + + assert!(matches!( + error, + Some(LibsyError::AlgorithmError { message }) + if message == "no modality-compatible random target has a positive weight" + )); + Ok(()) + } + #[tokio::test] async fn affinity_reuses_the_initial_random_selection() -> Result<()> { let names = ["a/model", "b/model"]; @@ -360,6 +519,59 @@ mod tests { Ok(()) } + #[tokio::test] + async fn incompatible_affinity_is_bypassed_for_a_multimodal_turn() -> Result<()> { + let names = ["text", "vision"]; + let affinity = Arc::new(AffinityRouter::new()); + let random = Arc::new(RandomClassifier::new( + target_set(&names), + Some(vec![100.0, 1.0]), + Some(42), + )?); + let algorithm: Arc = Arc::new( + FallThrough::<()>::new(target_set(&names)) + .with_name("affinity_random") + .with_target_modalities(target_modalities(&[ + ("text", &[InputModality::Text]), + ("vision", &[InputModality::Text, InputModality::Image]), + ])) + .with_processor(affinity.clone()) + .with_classifier(affinity) + .with_classifier(random), + ); + + let (_, first) = test_drive( + algorithm.clone(), + request_for_session("session-modalities"), + echo(), + ) + .await?; + let mut image = image_request(); + image.metadata = Some(Metadata { + session_id: Some("session-modalities".to_string()), + ..Metadata::default() + }); + let (_, second) = test_drive(algorithm, image, echo()).await?; + + assert_eq!( + first + .llm_response + .as_agg() + .map(completion_text) + .unwrap_or_default(), + "text" + ); + assert_eq!( + second + .llm_response + .as_agg() + .map(completion_text) + .unwrap_or_default(), + "vision" + ); + Ok(()) + } + #[test] fn rejects_invalid_weights() { let cases = [ diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index f751f7939..ac9168705 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -13,6 +13,7 @@ //! joined in unchanged — and then to the picker's default tier. The judge is //! asked per turn and its verdict is never pinned to the session. //! +use std::collections::BTreeSet; use std::sync::Arc; use async_trait::async_trait; @@ -28,7 +29,7 @@ use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier}; use crate::core::state::State; -use crate::{LibsyError, Result}; +use crate::{LibsyError, Result, TargetModalities}; use switchyard_protocol::{ModelId, Request, Response}; /// Telemetry name for a router this module assembles. @@ -63,6 +64,24 @@ impl Classifier for SourceStamp { } Ok((classification, served)) } + + async fn score_with_eligible_targets( + &self, + state: &mut State, + request: &mut Request, + driver: Option<&Driver>, + eligible_targets: &BTreeSet, + ) -> Result<(Classification, Option)> { + let (classification, served) = self + .inner + .score_with_eligible_targets(state, request, driver, eligible_targets) + .await?; + if let Some(winner) = classification.argmax(false)? { + record_decision_source(state, self.source); + record_routing_decision(self.source, &winner.target); + } + Ok((classification, served)) + } } /// The capability judge a stage router falls through to. @@ -131,6 +150,12 @@ impl StageRouter { route: build_route(capable, efficient, config)?, }) } + + /// Restricts stage decisions, judges, and fallback calls to compatible tiers. + pub fn with_target_modalities(mut self, target_modalities: TargetModalities) -> Self { + self.route = self.route.with_target_modalities(target_modalities); + self + } } #[async_trait] diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index b95f91f15..49ebc47ea 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -19,11 +19,14 @@ use std::collections::{HashMap, HashSet, hash_map::DefaultHasher}; use std::hash::{Hash, Hasher}; +use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use parking_lot::Mutex; -use switchyard_protocol::{ModelId, Request, Role}; +use serde::Serialize; +use serde_json::Value; +use switchyard_protocol::{ContentBlock, ModelId, Request, Role}; use crate::core::algorithm::{Driver, RoutingIdentity}; use crate::core::classifier::{Classification, Classifier, Score}; @@ -60,6 +63,29 @@ pub struct AffinityRouter { unkeyed_warning_emitted: AtomicBool, } +/// Retains one target for each user turn within a stable routing identity. +/// +/// An explicit agent turn id wins when present. Otherwise the turn is identified by the +/// ordinal and content of the latest human user message, excluding tool-loop continuation +/// items. Register one shared instance as both a processor and classifier. +#[derive(Default)] +pub(crate) struct TurnAffinityRouter { + assignments: Mutex>, + unkeyed_warning_emitted: AtomicBool, +} + +#[derive(Clone, Hash, PartialEq, Eq)] +struct TurnRoutingIdentity { + routing: RoutingIdentity, + turn: TurnIdentity, +} + +#[derive(Clone, Hash, PartialEq, Eq)] +enum TurnIdentity { + Explicit(String), + UserMessage { ordinal: usize, digest: u64 }, +} + impl AffinityRouter { /// Creates a router that latches every decision. pub fn new() -> Self { @@ -140,6 +166,27 @@ impl AffinityRouter { } } +impl TurnAffinityRouter { + /// Creates a router that latches decisions only for the current user turn. + pub(crate) fn new() -> Self { + Self::default() + } + + fn affinity_key(&self, request: &Request) -> Option { + let key = RoutingIdentity::from_request(request).and_then(|routing| { + turn_identity(request).map(|turn| TurnRoutingIdentity { routing, turn }) + }); + if key.is_none() && !self.unkeyed_warning_emitted.swap(true, Ordering::Relaxed) { + tracing::warn!( + target: "libsy", + "turn affinity is enabled but this request has no usable session and user-turn \ + identity, so no turn affinity is applied" + ); + } + key + } +} + #[async_trait] impl Processor for AffinityRouter where @@ -160,6 +207,27 @@ where } } +#[async_trait] +impl Processor for TurnAffinityRouter +where + S: Send + 'static, +{ + async fn process(&self, _state: &mut S, event: Event<'_>) -> crate::Result<()> { + if let Event::Decision { request, decision } = event + && let Some(key) = self.affinity_key(request) + { + let mut assignments = self.assignments.lock(); + if !assignments.contains_key(&key) { + evict_if_full(&mut assignments); + } + // Replacing an assignment is intentional: modality filtering may make the + // retained target ineligible and force a compatible decision for this turn. + assignments.insert(key, decision.selected_model_id().clone()); + } + Ok(()) + } +} + /// Hashes the first user message so later turns retain the initial task's affinity. /// For benchmarking purpose with harnesses, task instructions are added as a user prompt to the request so we hash the initial user message. /// TODO: Have not considered multi-modal payloads yet. That needs to be handled separately. @@ -174,6 +242,123 @@ fn first_user_message_hash(request: &Request) -> Option { Some(format!("{:016x}", hasher.finish())) } +/// Whether a normalized user-role message is authored input rather than tool-loop state. +pub(crate) fn is_user_turn_message(message: &switchyard_protocol::Message) -> bool { + message.role == Role::User + && !matches!( + message.content.as_slice(), + [ContentBlock::Text { text }] + if text.starts_with("Tool result ") && text.contains(": ") + ) + && message.content.iter().any(|block| { + matches!( + block, + ContentBlock::Text { .. } + | ContentBlock::Image { .. } + | ContentBlock::Audio { .. } + | ContentBlock::Video { .. } + | ContentBlock::File { .. } + ) + }) +} + +fn turn_identity(request: &Request) -> Option { + if let Some(turn_id) = request + .metadata + .as_ref() + .and_then(|metadata| metadata.turn_id.as_deref()) + .filter(|turn_id| !turn_id.is_empty()) + { + return Some(TurnIdentity::Explicit(turn_id.to_string())); + } + + raw_user_turn(request) + .or_else(|| normalized_user_turn(request)) + .map(|(ordinal, digest)| TurnIdentity::UserMessage { ordinal, digest }) +} + +/// Reads human user turns from the original body so provider-native tool items cannot be +/// mistaken for a follow-up merely because their normalized fallback role is `user`. +fn raw_user_turn(request: &Request) -> Option<(usize, u64)> { + let body = request.raw_request.as_ref()?; + if let Some(input) = body.get("input") { + return match input { + Value::String(_) => content_digest(input).map(|digest| (1, digest)), + Value::Array(items) => latest_raw_user_turn(items), + _ => None, + }; + } + body.get("messages") + .and_then(Value::as_array) + .and_then(|messages| latest_raw_user_turn(messages)) +} + +fn latest_raw_user_turn(items: &[Value]) -> Option<(usize, u64)> { + let mut ordinal = 0; + let mut latest = None; + for item in items { + let Some(object) = item.as_object() else { + continue; + }; + if object.get("role").and_then(Value::as_str) != Some("user") { + continue; + } + let item_type = object.get("type").and_then(Value::as_str); + if item_type.is_some_and(|kind| kind != "message") { + continue; + } + let content = object.get("content").unwrap_or(&Value::Null); + if raw_content_is_only_tool_results(&content) { + continue; + } + ordinal += 1; + latest = content_digest(content); + } + latest.map(|digest| (ordinal, digest)) +} + +fn raw_content_is_only_tool_results(content: &Value) -> bool { + let Some(blocks) = content.as_array() else { + return false; + }; + !blocks.is_empty() + && blocks + .iter() + .all(|block| block.get("type").and_then(Value::as_str) == Some("tool_result")) +} + +fn normalized_user_turn(request: &Request) -> Option<(usize, u64)> { + let mut ordinal = 0; + let mut latest = None; + for message in &request.llm_request.messages { + if !is_user_turn_message(message) { + continue; + } + ordinal += 1; + latest = content_digest(&message.content); + } + latest.map(|digest| (ordinal, digest)) +} + +fn content_digest(content: &impl Serialize) -> Option { + let mut hasher = DefaultHasher::new(); + serde_json::to_writer(HashWriter(&mut hasher), content).ok()?; + Some(hasher.finish()) +} + +struct HashWriter<'a>(&'a mut DefaultHasher); + +impl Write for HashWriter<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + Hasher::write(self.0, buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + #[async_trait] impl Classifier for AffinityRouter where @@ -202,8 +387,38 @@ where } } +#[async_trait] +impl Classifier for TurnAffinityRouter +where + S: Send + 'static, +{ + async fn score( + &self, + _state: &mut S, + request: &mut Request, + _driver: Option<&Driver>, + ) -> crate::Result<(Classification, Option)> { + let assigned = self + .affinity_key(request) + .and_then(|key| self.assignments.lock().get(&key).cloned()); + Ok(( + Classification::Scores(match assigned { + Some(target) => vec![Score { + confidence: 1.0, + target, + }], + None => Vec::new(), + }), + None, + )) + } +} + /// Evicts one arbitrary assignment when the map has reached [`MAX_ASSIGNMENTS`]. -fn evict_if_full(assignments: &mut HashMap) { +fn evict_if_full(assignments: &mut HashMap) +where + K: Clone + Eq + Hash, +{ if assignments.len() >= MAX_ASSIGNMENTS && let Some(evicted) = assignments.keys().next().cloned() { @@ -767,4 +982,94 @@ mod tests { ); Ok(()) } + + fn responses_turn(input: Value) -> Request { + Request { + llm_request: text_request(Some("auto".to_string()), "task"), + raw_request: Some(serde_json::json!({ + "model": "auto", + "input": input, + })), + metadata: Some(Metadata { + session_id: Some("turn-session".to_string()), + ..Metadata::default() + }), + } + } + + #[tokio::test] + async fn turn_affinity_ignores_responses_tool_loop_items_and_resets_on_user_input() + -> Result<(), BoxErr> { + let router = TurnAffinityRouter::new(); + let mut state = (); + let mut first = responses_turn(serde_json::json!([ + {"role": "user", "content": "Implement the parser."} + ])); + router + .process( + &mut state, + Event::Decision { + request: &mut first, + decision: &fixed_decision("model-a"), + }, + ) + .await?; + + let mut continuation = responses_turn(serde_json::json!([ + {"role": "user", "content": "Implement the parser."}, + {"type": "function_call", "call_id": "call-1", "name": "read", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call-1", "output": "source"}, + {"type": "tool_search_call", "call_id": "search-1", "status": "completed"} + ])); + assert_eq!( + scores(&router, &mut state, &mut continuation) + .await? + .first() + .map(|score| score.target.as_str()), + Some("model-a") + ); + + let mut follow_up = responses_turn(serde_json::json!([ + {"role": "user", "content": "Implement the parser."}, + {"type": "function_call", "call_id": "call-1", "name": "read", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call-1", "output": "source"}, + {"role": "user", "content": "Now add regression tests."} + ])); + assert!( + scores(&router, &mut state, &mut follow_up) + .await? + .is_empty() + ); + Ok(()) + } + + #[tokio::test] + async fn turn_affinity_can_replace_an_incompatible_assignment() -> Result<(), BoxErr> { + let router = TurnAffinityRouter::new(); + let mut state = (); + let mut request = responses_turn(serde_json::json!([ + {"role": "user", "content": "Inspect the tool output."} + ])); + + for model in ["text-model", "vision-model"] { + router + .process( + &mut state, + Event::Decision { + request: &mut request, + decision: &fixed_decision(model), + }, + ) + .await?; + } + + assert_eq!( + scores(&router, &mut state, &mut request) + .await? + .first() + .map(|score| score.target.as_str()), + Some("vision-model") + ); + Ok(()) + } } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index f46b4815e..b8024e31e 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -236,6 +236,21 @@ pub async fn drive( where F: Fn(CallModel) -> Fut, Fut: Future>, +{ + drive_with_decision_observer(algorithm, request, serve, |_| {}).await +} + +/// Drives an algorithm like [`drive`] and observes each decision before its model call. +pub async fn drive_with_decision_observer( + algorithm: Arc, + request: Request, + serve: F, + observe_decision: O, +) -> Result<(Vec, Response)> +where + F: Fn(CallModel) -> Fut, + Fut: Future>, + O: Fn(&Decision), { let stream = algorithm.run_stream(request); tokio::pin!(stream); @@ -255,7 +270,10 @@ where None => break, // stream has ended, no more steps Some(item) => match item? { Step::CallModel(call) => in_flight.push(serve(*call)), - Step::Decision(decision) => trace.push(decision), + Step::Decision(decision) => { + observe_decision(&decision); + trace.push(decision); + } Step::Done(response) => { final_response = Some(*response); break; diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs index a515a7c64..528916610 100644 --- a/crates/libsy/src/core/classifier.rs +++ b/crates/libsy/src/core/classifier.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::collections::BTreeSet; + use crate::core::algorithm::Driver; use crate::{LibsyError, Result}; use async_trait::async_trait; @@ -43,6 +45,20 @@ impl Classification { } } } + + /// Removes scores for targets unavailable to the current request. + pub(crate) fn retain_eligible(self, eligible_targets: &BTreeSet) -> Self { + let retain = |scores: Vec| { + scores + .into_iter() + .filter(|score| eligible_targets.contains(&score.target)) + .collect() + }; + match self { + Self::Scores(scores) => Self::Scores(retain(scores)), + Self::Ambiguous(scores) => Self::Ambiguous(retain(scores)), + } + } } /// The highest-confidence score, or `None` when the set is empty (the classifier abstained). @@ -76,6 +92,11 @@ pub trait Classifier: Send + Sync { None } + /// Whether this classifier must score when modality filtering leaves one eligible target. + fn needs_single_eligible_scoring(&self) -> bool { + false + } + /// Score the classifier's targets given the current state and request. /// /// When present, `driver` lets a classifier offload model calls. It is `None` @@ -92,6 +113,25 @@ pub trait Classifier: Send + Sync { request: &mut Request, driver: Option<&Driver>, ) -> Result<(Classification, Option)>; + + /// Scores with the completion targets eligible for this request. + /// + /// The default delegates to [`score`](Self::score), preserving existing + /// classifier implementations. Classifiers whose selection itself has side + /// effects, such as weighted sampling or pre-calling a tier, can override + /// this hook to avoid work for ineligible targets. + async fn score_with_eligible_targets( + &self, + state: &mut S, + request: &mut Request, + driver: Option<&Driver>, + _eligible_targets: &BTreeSet, + ) -> Result<(Classification, Option)> + where + S: Send, + { + self.score(state, request, driver).await + } } #[cfg(test)] diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs index 7cb47b06d..9cd4a8364 100644 --- a/crates/libsy/src/error.rs +++ b/crates/libsy/src/error.rs @@ -5,9 +5,13 @@ use std::error::Error as StdError; -use switchyard_protocol::{LlmClientError, ModelId}; +use std::collections::BTreeSet; + +use switchyard_protocol::{InputModality, LlmClientError, ModelId}; use thiserror::Error; +use crate::TargetModalities; + /// Result type returned by libsy APIs. pub type Result = std::result::Result; @@ -25,6 +29,17 @@ pub enum LibsyError { #[error("no routing targets are configured")] NoTargets, + /// No completion target accepts every modality present in the request. + #[error( + "no compatible targets for required input modalities {required_modalities:?}; candidate capabilities: {target_modalities:?}" + )] + NoCompatibleTargets { + /// Modalities required by the normalized request. + required_modalities: BTreeSet, + /// Declared capability sets for the route's completion targets. + target_modalities: TargetModalities, + }, + /// An algorithm could not complete for an algorithm-specific reason. #[error("{message}")] AlgorithmError { diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 583ce1f13..1810c186f 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -5,7 +5,9 @@ #![doc = include_str!("../README.md")] mod core; -pub use core::algorithm::{Algorithm, CallModel, Driver, Step, StepStream, drive}; +pub use core::algorithm::{ + Algorithm, CallModel, Driver, Step, StepStream, drive, drive_with_decision_observer, +}; pub use core::classifier::{Classification, Classifier, Score}; pub use core::processor::{Event, Processor}; pub use core::state::{State, StateValue}; @@ -13,6 +15,9 @@ pub use core::state::{State, StateValue}; mod error; pub use error::{DriverError, LibsyError, Result}; +mod target_modalities; +pub use target_modalities::TargetModalities; + mod algorithms; pub use algorithms::llm_class::{ CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier, diff --git a/crates/libsy/src/target_modalities.rs b/crates/libsy/src/target_modalities.rs new file mode 100644 index 000000000..883a0cda9 --- /dev/null +++ b/crates/libsy/src/target_modalities.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared target-capability map and request eligibility checks. + +use std::collections::{BTreeMap, BTreeSet}; + +use switchyard_protocol::{InputModality, ModelId, Request}; + +use crate::{LibsyError, Result}; + +/// Input modalities supported by each completion target in one route. +/// +/// A router without this map retains legacy behavior. When present, the map is +/// expected to cover every completion target in the route. +pub type TargetModalities = BTreeMap>; + +/// Returns compatible targets in configured route order. +pub(crate) fn eligible_targets( + targets: &[ModelId], + target_modalities: &TargetModalities, + request: &Request, +) -> Result<(BTreeSet, Vec)> { + let required_modalities = request.llm_request.input_modalities(); + let eligible = targets + .iter() + .filter(|target| { + target_modalities + .get(*target) + .is_some_and(|supported| required_modalities.is_subset(supported)) + }) + .cloned() + .collect::>(); + + if eligible.is_empty() { + let target_modalities = targets + .iter() + .filter_map(|target| { + target_modalities + .get(target) + .cloned() + .map(|modalities| (target.clone(), modalities)) + }) + .collect(); + return Err(LibsyError::NoCompatibleTargets { + required_modalities, + target_modalities, + }); + } + + Ok((required_modalities, eligible)) +} diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index c2386c23f..f559340ec 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -3,7 +3,8 @@ //! Provider-neutral conversation types shared by routing, clients, and translation. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -72,6 +73,43 @@ impl Message { } } +/// Provider-neutral input types that a completion target may accept. +/// +/// Declaration order is the canonical order used for discovery and diagnostics. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InputModality { + /// Text, reasoning, and refusal content. + Text, + /// Image content. + Image, + /// Audio content. + Audio, + /// Video content. + Video, + /// File content. + File, +} + +impl InputModality { + /// Returns the stable configuration and discovery name for this modality. + pub const fn as_str(self) -> &'static str { + match self { + Self::Text => "text", + Self::Image => "image", + Self::Audio => "audio", + Self::Video => "video", + Self::File => "file", + } + } +} + +impl fmt::Display for InputModality { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + /// Normalized content block variants carried by messages and tool results. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] @@ -329,6 +367,52 @@ pub struct LlmRequest { pub preservation: PreservationMetadata, } +impl LlmRequest { + /// Returns every typed input modality required by this request. + /// + /// Instructions, conversation history, and nested tool-result content all count. + /// Tool calls and unknown provider extension blocks do not imply a modality. + pub fn input_modalities(&self) -> BTreeSet { + let mut modalities = BTreeSet::new(); + for instruction in &self.instructions { + collect_input_modalities(&instruction.content, &mut modalities); + } + for message in &self.messages { + collect_input_modalities(&message.content, &mut modalities); + } + modalities + } +} + +/// Walks normalized content recursively so media returned by tools remains routable. +fn collect_input_modalities(content: &[ContentBlock], modalities: &mut BTreeSet) { + for block in content { + match block { + ContentBlock::Text { .. } + | ContentBlock::Reasoning { .. } + | ContentBlock::Refusal { .. } => { + modalities.insert(InputModality::Text); + } + ContentBlock::Image { .. } => { + modalities.insert(InputModality::Image); + } + ContentBlock::Audio { .. } => { + modalities.insert(InputModality::Audio); + } + ContentBlock::Video { .. } => { + modalities.insert(InputModality::Video); + } + ContentBlock::File { .. } => { + modalities.insert(InputModality::File); + } + ContentBlock::ToolResult(result) => { + collect_input_modalities(&result.content, modalities); + } + ContentBlock::ToolCall(_) | ContentBlock::Unknown { .. } => {} + } + } +} + /// Normalized token usage counts. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Usage { @@ -490,4 +574,109 @@ mod tests { ); Ok(()) } + + #[test] + fn input_modalities_cover_instructions_history_and_nested_tool_results() { + let request = LlmRequest { + instructions: vec![InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Reasoning { + text: "think carefully".to_string(), + signature: None, + }], + }], + messages: vec![ + Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "compare these inputs".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/image.png".to_string(), + detail: None, + }, + }, + ContentBlock::Audio { + source: MediaSource::Url { + url: "https://example.test/audio.wav".to_string(), + media_type: Some("audio/wav".to_string()), + }, + }, + ], + }, + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".to_string(), + content: vec![ + ContentBlock::Video { + source: MediaSource::Base64 { + media_type: Some("video/mp4".to_string()), + data: "AAAA".to_string(), + }, + }, + ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-2".to_string(), + content: vec![ContentBlock::File { + source: FileSource::FileId("file-1".to_string()), + }], + is_error: None, + }), + ], + is_error: Some(false), + })], + }, + ], + ..LlmRequest::default() + }; + + assert_eq!( + request.input_modalities(), + BTreeSet::from([ + InputModality::Text, + InputModality::Image, + InputModality::Audio, + InputModality::Video, + InputModality::File, + ]) + ); + } + + #[test] + fn input_modalities_deduplicate_blocks_and_ignore_untyped_extensions() { + let request = LlmRequest { + messages: vec![Message { + role: Role::Assistant, + content: vec![ + ContentBlock::Refusal { + text: "no".to_string(), + }, + ContentBlock::Text { + text: String::new(), + }, + ContentBlock::ToolCall(ToolCall { + id: "call-1".to_string(), + name: "inspect_image".to_string(), + arguments: json!({"image": "not-a-typed-media-block"}), + }), + ContentBlock::Unknown { + provider: FormatId::from("extension"), + raw: json!({"type": "input_image"}), + }, + ], + }], + extensions: ProviderExtensions { + fields: Map::from_iter([("input_audio".to_string(), json!(true))]), + }, + ..LlmRequest::default() + }; + + assert_eq!( + request.input_modalities(), + BTreeSet::from([InputModality::Text]) + ); + assert!(LlmRequest::default().input_modalities().is_empty()); + } } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index f52e29e36..05ba57bf3 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -65,6 +65,7 @@ impl PyTaskClassifierConfig { *, threshold_step=0.0, session_affinity=false, + turn_affinity=false, message_hash_fallback=false, recent_turn_window=None, max_output_tokens=4096, @@ -76,6 +77,7 @@ impl PyTaskClassifierConfig { base_threshold: f64, threshold_step: f64, session_affinity: bool, + turn_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, max_output_tokens: u64, @@ -101,6 +103,7 @@ impl PyTaskClassifierConfig { base_threshold, threshold_step, session_affinity, + turn_affinity, message_hash_fallback, recent_turn_window, contract, diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs index 862eea2ad..d3e2dbc80 100644 --- a/crates/switchyard-py/src/server_bindings.rs +++ b/crates/switchyard-py/src/server_bindings.rs @@ -26,6 +26,7 @@ const DEFAULT_SHUTDOWN_TIMEOUT_SECS: f64 = 2.0; struct PyServer { addr: SocketAddr, caller_auth_by_model: HashMap>, + input_modalities_by_model: HashMap>, shutdown: Option>, completion: Option>>, task: Option>, @@ -48,6 +49,21 @@ impl PyServer { }) .collect::>>() .map_err(server_error)?; + let input_modalities_by_model = state + .models() + .map(|model| { + state.input_modalities(model).map(|modalities| { + ( + model.to_string(), + modalities + .into_iter() + .map(|modality| modality.as_str().to_string()) + .collect(), + ) + }) + }) + .collect::>>() + .map_err(server_error)?; let runtime = pyo3_async_runtimes::tokio::get_runtime(); let server = { let _guard = runtime.enter(); @@ -77,6 +93,7 @@ impl PyServer { Ok(Self { addr, caller_auth_by_model, + input_modalities_by_model, shutdown: Some(shutdown), completion: Some(completion), task: Some(task), @@ -103,6 +120,14 @@ impl PyServer { .ok_or_else(|| PyValueError::new_err(format!("unknown route model {model:?}"))) } + /// Returns the input modalities advertised for a route model. + fn input_modalities(&self, model: &str) -> PyResult> { + self.input_modalities_by_model + .get(model) + .cloned() + .ok_or_else(|| PyValueError::new_err(format!("unknown route model {model:?}"))) + } + /// Gracefully stops the server and flushes pending telemetry. #[pyo3(signature = (timeout_secs=DEFAULT_SHUTDOWN_TIMEOUT_SECS))] fn close(&mut self, py: Python<'_>, timeout_secs: f64) -> PyResult<()> { diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index b853929d7..61dce32d1 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -91,14 +91,23 @@ for equal weighting. The optional `seed` reproduces the selection sequence for t ## Session routing log -Pass `--routing-log-file PATH` to append one JSON record after each completed routed response. -Streaming responses are recorded after the stream drains. When enabled, +Pass `--routing-log-file PATH` to append JSON records for `request_start`, routing +`decision`, upstream `start`, `ttfb`, `usage`, and `completion` events. Completion records do not depend on +provider usage and report `ok`, `error`, or `cancelled`; streaming responses complete when +the stream drains or is dropped. When enabled, `GET /v1/routing/session-stats?session_id=ID` rescans the durable log and returns call and token -totals for that normalized session ID, normally supplied as `x-switchyard-session-id`, grouped by +totals from `usage` records for that normalized session ID, normally supplied as +`x-switchyard-session-id`, grouped by served model. The legacy `proxy_x_session_id` remains a fallback when no normalized session ID is present. The endpoint returns `404` when the session has no records and is not registered when routing logging is disabled. +Add `--routing-log-classifier-content` to append a `classifier_content` event containing the +normalized classifier request, provider-returned reasoning, and raw verdict. This is opt-in +because the classifier request and reasoning can repeat user-provided secrets. Transport headers +and provider credentials are never included. Store this log with restricted permissions and +rotation appropriate for sensitive request content. + An `llm_classifier` route sends each task to `classifier_target` for a capability verdict, then routes to `weak_target` or `strong_target`. Beyond the three targets it accepts these keys; only `base_threshold` is required, and anything the judge cannot decide routes to `strong_target`: @@ -108,6 +117,7 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts | `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. | | `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `session_affinity` | `false` | Reuses a session's first routing decision on later turns, so the judge is called once per session rather than once per turn. | +| `turn_affinity` | `false` | Reuses a decision across tool-loop continuations until the next human user message. Requires a session identity and cannot be combined with `session_affinity`; modality filtering can replace an incompatible assignment. | | `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `session_affinity = true`. | Session affinity retains a decision for the process lifetime, including a `strong_target` diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 231d4e2bc..480a7e02f 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,6 +52,10 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, + /// Include classifier prompts, model reasoning, and verdicts in the routing log. + #[arg(long, requires = "routing_log_file")] + routing_log_classifier_content: bool, + /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] tls_cert: Option, @@ -72,6 +76,7 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } + state = state.with_routing_log_classifier_content(self.routing_log_classifier_content); let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { if !cert.exists() || !key.exists() { diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 561390235..80ad5c80e 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -3,7 +3,7 @@ //! Typed TOML configuration and explicit construction for the Rust server. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::path::Path; use std::sync::Arc; @@ -12,7 +12,7 @@ use libsy::{ Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, - StageRouterConfig, TargetPrompts, TaskClassifierConfig, + StageRouterConfig, TargetModalities, TargetPrompts, TaskClassifierConfig, }; use serde::Deserialize; use serde_json::Value; @@ -20,7 +20,7 @@ use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::{ModelId, RoutedLlmClient}; +use switchyard_protocol::{InputModality, ModelId, RoutedLlmClient}; use crate::{ CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState, @@ -79,6 +79,7 @@ impl ServerConfig { for (target_name, target) in &self.targets { validate_value("target name", target_name)?; validate_value(&format!("target {target_name} id"), &target.id)?; + validate_input_modalities(target_name, target.input_modalities.as_deref())?; if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) { tracing::warn!( "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.", @@ -94,13 +95,15 @@ impl ServerConfig { for (route_name, config) in &self.routes { validate_value("route name", route_name)?; validate_value(&format!("route {route_name} id"), config.id())?; - let capabilities = config.capabilities(); + let (target_modalities, input_modalities) = + self.route_modalities(route_name, config)?; + let capabilities = config.capabilities(input_modalities); if capabilities.context_window == Some(0) { return Err(ServerError::new(format!( "route {route_name} context_window must be greater than zero" ))); } - let algorithm = build_algorithm(route_name, config, &targets)?; + let algorithm = build_algorithm(route_name, config, &targets, target_modalities)?; let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?; let count_tokens_target = self.build_count_tokens_target(config, &clients); routes.push(( @@ -115,6 +118,75 @@ impl ServerConfig { ServerState::new_with_capabilities(routes) } + /// Validates per-route declaration completeness and derives routing/discovery metadata. + fn route_modalities( + &self, + route_name: &str, + route: &RouteConfig, + ) -> ServerResult<(Option, Vec)> { + for judge_name in route.judge_target_names() { + let judge = self.targets.get(judge_name).ok_or_else(|| { + ServerError::new(format!("route references unknown target {judge_name}")) + })?; + if let Some(modalities) = &judge.input_modalities + && !modalities.contains(&InputModality::Text) + { + return Err(ServerError::new(format!( + "judge target {judge_name} for route {route_name} must include text in input_modalities" + ))); + } + } + + let target_names = route.routing_target_names(); + if target_names.is_empty() { + return Ok((None, vec![InputModality::Text])); + } + let targets = target_names + .iter() + .map(|name| { + self.targets + .get(*name) + .map(|target| (*name, target)) + .ok_or_else(|| { + ServerError::new(format!("route references unknown target {name}")) + }) + }) + .collect::>>()?; + let declared = targets + .iter() + .filter(|(_, target)| target.input_modalities.is_some()) + .count(); + if declared == 0 { + return Ok((None, vec![InputModality::Text])); + } + if declared != targets.len() { + return Err(ServerError::new(format!( + "route {route_name} must declare input_modalities for every completion target or none" + ))); + } + + let mut target_modalities = TargetModalities::new(); + let mut advertised = BTreeSet::new(); + for (_, target) in targets { + let modalities = target + .input_modalities + .as_ref() + .map(|modalities| modalities.iter().copied().collect::>()) + .ok_or_else(|| ServerError::new("validated target modalities were missing"))?; + advertised.extend(modalities.iter().copied()); + if let Some(existing) = target_modalities.get(&target.id) + && existing != &modalities + { + return Err(ServerError::new(format!( + "route {route_name} maps model id {} to conflicting input_modalities", + target.id + ))); + } + target_modalities.insert(target.id.clone(), modalities); + } + Ok((Some(target_modalities), advertised.into_iter().collect())) + } + fn build_clients(&self) -> ServerResult>> { let mut models_by_client = self .llm_clients @@ -137,7 +209,12 @@ impl ServerConfig { .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; model_configs.push(ModelConfig::new( target.id.clone(), - build_backend(&target.llm_client, client_config, &target.extra_body)?, + build_backend( + &target.llm_client, + client_config, + &target.extra_body, + target.reasoning_effort_override.as_deref(), + )?, None, )); } @@ -256,6 +333,12 @@ struct LlmClientConfig { extra_headers: BTreeMap, #[serde(default = "default_max_retries")] max_retries: u32, + #[serde(default)] + bridge_custom_tools: bool, + #[serde(default)] + eager_load_tool_search: bool, + #[serde(default)] + xai_responses_compatibility: bool, } #[derive(Debug, Deserialize)] @@ -264,7 +347,10 @@ struct TargetConfig { id: ModelId, llm_client: String, #[serde(default)] + input_modalities: Option>, + #[serde(default)] extra_body: BTreeMap, + reasoning_effort_override: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -322,6 +408,7 @@ struct CapabilityClassifierRouteConfig { base_threshold: f64, threshold_step: f64, session_affinity: bool, + turn_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, prompt: Option, @@ -347,8 +434,10 @@ struct CustomClassifierRouteConfig { response_schema: String, policy: ClassifierPolicyConfig, session_affinity: bool, + turn_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_text_only: bool, max_output_tokens: u64, } @@ -363,6 +452,8 @@ enum RouteConfig { tool_calling: Option, #[serde(default)] reasoning: Option, + #[serde(default)] + web_search: Option, }, Random { id: ModelId, @@ -372,6 +463,8 @@ enum RouteConfig { tool_calling: Option, #[serde(default)] reasoning: Option, + #[serde(default)] + web_search: Option, targets: Vec, weights: Option>, seed: Option, @@ -384,6 +477,8 @@ enum RouteConfig { tool_calling: Option, #[serde(default)] reasoning: Option, + #[serde(default)] + web_search: Option, target: String, }, LlmClassifier { @@ -394,6 +489,10 @@ enum RouteConfig { tool_calling: Option, #[serde(default)] reasoning: Option, + #[serde(default)] + web_search: Option, + #[serde(default = "enabled_by_default")] + target_failover: bool, classifier_target: String, #[serde(default)] mode: Option, @@ -408,10 +507,14 @@ enum RouteConfig { #[serde(default)] session_affinity: bool, #[serde(default)] + turn_affinity: bool, + #[serde(default)] message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, #[serde(default)] + judge_text_only: bool, + #[serde(default)] prompt: Option, #[serde(default)] response_format_type: ClassifierResponseFormat, @@ -436,6 +539,8 @@ enum RouteConfig { tool_calling: Option, #[serde(default)] reasoning: Option, + #[serde(default)] + web_search: Option, capable_target: String, efficient_target: String, /// Tier a turn falls back to when the signals are not confident. @@ -487,6 +592,7 @@ impl StageClassifierConfig { base_threshold: self.base_threshold, threshold_step: self.threshold_step, session_affinity: self.session_affinity, + turn_affinity: false, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, contract: classifier_contract(self.prompt.as_deref()) @@ -552,55 +658,67 @@ impl RouteConfig { /// a classifier also calls its judge, and that call needs a client too. fn callable_target_names(&self) -> Vec<&str> { let mut names = self.routing_target_names(); + names.extend(self.judge_target_names()); + names + } + + /// Targets used only to construct routing verdicts. + fn judge_target_names(&self) -> Vec<&str> { match self { Self::LlmClassifier { classifier_target, .. - } => names.push(classifier_target), + } => vec![classifier_target], Self::StageRouter { classifier: Some(classifier), .. - } => names.push(&classifier.target), - _ => {} + } => vec![&classifier.target], + _ => Vec::new(), } - names } - fn capabilities(&self) -> ModelCapabilities { + fn capabilities(&self, input_modalities: Vec) -> ModelCapabilities { use RouteConfig::*; match self { Noop { context_window, tool_calling, reasoning, + web_search, .. } | Random { context_window, tool_calling, reasoning, + web_search, .. } | Passthrough { context_window, tool_calling, reasoning, + web_search, .. } | LlmClassifier { context_window, tool_calling, reasoning, + web_search, .. } | StageRouter { context_window, tool_calling, reasoning, + web_search, .. } => ModelCapabilities { context_window: *context_window, tool_calling: *tool_calling, reasoning: *reasoning, + web_search: *web_search, + input_modalities, }, } } @@ -613,8 +731,10 @@ impl RouteConfig { base_threshold, threshold_step, session_affinity, + turn_affinity, message_hash_fallback, recent_turn_window, + judge_text_only, prompt, response_format_type, max_output_tokens, @@ -637,6 +757,13 @@ impl RouteConfig { match selected_mode { ClassifierMode::Capability => { + if *judge_text_only { + return Err(classifier_field_error( + route_name, + "judge_text_only", + "capability", + )); + } if escalation.is_some() { return Err(classifier_field_error( route_name, @@ -671,6 +798,7 @@ impl RouteConfig { )?, threshold_step: threshold_step.unwrap_or_default(), session_affinity: *session_affinity, + turn_affinity: *turn_affinity, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, prompt: prompt.clone(), @@ -680,6 +808,13 @@ impl RouteConfig { )) } ClassifierMode::Escalation => { + if *judge_text_only { + return Err(classifier_field_error( + route_name, + "judge_text_only", + "escalation", + )); + } reject_custom_fields( route_name, "escalation", @@ -692,6 +827,7 @@ impl RouteConfig { && (base_threshold.is_some() || threshold_step.is_some() || *session_affinity + || *turn_affinity || *message_hash_fallback || recent_turn_window.is_some()) { @@ -746,8 +882,10 @@ impl RouteConfig { )?, policy: required_classifier_field(route_name, "policy", policy)?, session_affinity: *session_affinity, + turn_affinity: *turn_affinity, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_text_only: *judge_text_only, max_output_tokens: *max_output_tokens, }, )) @@ -798,6 +936,7 @@ fn build_backend( client_name: &str, config: &LlmClientConfig, extra_body: &BTreeMap, + reasoning_effort_override: Option<&str>, ) -> ServerResult { let base_url = config.base_url.trim(); if base_url.is_empty() { @@ -815,6 +954,22 @@ fn build_backend( "llm client {client_name} cannot set both forward_auth and api_key_env" ))); } + if config.bridge_custom_tools && !matches!(config.format, ClientFormat::OpenAiResponses) { + return Err(ServerError::new(format!( + "llm client {client_name} bridge_custom_tools requires format openai_responses" + ))); + } + if config.eager_load_tool_search && !matches!(config.format, ClientFormat::OpenAiResponses) { + return Err(ServerError::new(format!( + "llm client {client_name} eager_load_tool_search requires format openai_responses" + ))); + } + if config.xai_responses_compatibility && !matches!(config.format, ClientFormat::OpenAiResponses) + { + return Err(ServerError::new(format!( + "llm client {client_name} xai_responses_compatibility requires format openai_responses" + ))); + } let api_key = config .api_key_env .as_deref() @@ -837,12 +992,27 @@ fn build_backend( Ok(api_key) }) .transpose()?; + let reasoning_effort_override = reasoning_effort_override + .map(|effort| { + let effort = effort.trim(); + if effort.is_empty() { + return Err(ServerError::new(format!( + "llm client {client_name} reasoning_effort_override must not be empty" + ))); + } + Ok(effort.to_string()) + }) + .transpose()?; let http = HttpBackendConfig { base_url: base_url.to_string(), api_key, forward_auth: config.forward_auth, extra_headers: config.extra_headers.clone(), extra_body: extra_body.clone(), + reasoning_effort_override, + bridge_custom_tools: config.bridge_custom_tools, + eager_load_tool_search: config.eager_load_tool_search, + xai_responses_compatibility: config.xai_responses_compatibility, max_retries: config.max_retries, }; Ok(match config.format { @@ -860,6 +1030,7 @@ fn build_algorithm( route_name: &str, config: &RouteConfig, targets: &BTreeMap, + target_modalities: Option, ) -> ServerResult> { match config { RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})), @@ -871,20 +1042,29 @@ fn build_algorithm( } => { let target_set = resolve_targets(route_name, names.iter().map(String::as_str), targets)?; - let algorithm = Random::new(target_set, weights.clone(), *seed) + let mut algorithm = Random::new(target_set, weights.clone(), *seed) .map_err(|error| ServerError::new(format!("random route {route_name}: {error}")))?; + if let Some(target_modalities) = target_modalities { + algorithm = algorithm.with_target_modalities(target_modalities); + } Ok(Arc::new(algorithm)) } RouteConfig::Passthrough { target, .. } => { let target = resolve_target_model_id(route_name, target, targets)?; - Ok(Arc::new(Passthrough::new(target))) + let mut algorithm = Passthrough::new(target); + if let Some(target_modalities) = target_modalities { + algorithm = algorithm.with_target_modalities(target_modalities); + } + Ok(Arc::new(algorithm)) } RouteConfig::LlmClassifier { - classifier_target, .. + classifier_target, + target_failover, + .. } => { let classifier = resolve_target_model_id(route_name, classifier_target, targets)?; let mode = config.classifier_mode(route_name)?; - let algorithm = match mode { + let mut algorithm = match mode { LlmClassifierModeConfig::Capability(config) => { let strong = resolve_target_model_id(route_name, &config.strong_target, targets)?; @@ -893,6 +1073,7 @@ fn build_algorithm( base_threshold: config.base_threshold, threshold_step: config.threshold_step, session_affinity: config.session_affinity, + turn_affinity: config.turn_affinity, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, contract: classifier_contract(config.prompt.as_deref()) @@ -942,8 +1123,10 @@ fn build_algorithm( config.policy.into_libsy(), ); classifier_config.session_affinity = config.session_affinity; + classifier_config.turn_affinity = config.turn_affinity; classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_text_only = config.judge_text_only; classifier_config.max_output_tokens = config.max_output_tokens; LlmTaskClassifier::new(LlmClassifierConfig::Custom { judge_target: classifier, @@ -956,6 +1139,10 @@ fn build_algorithm( .map_err(|error| { ServerError::new(format!("llm_classifier route {route_name}: {error}")) })?; + algorithm = algorithm.with_target_failover(*target_failover); + if let Some(target_modalities) = target_modalities { + algorithm = algorithm.with_target_modalities(target_modalities); + } Ok(Arc::new(algorithm)) } RouteConfig::StageRouter { @@ -999,9 +1186,12 @@ fn build_algorithm( ) }) .transpose()?; - let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { + let mut algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { ServerError::new(format!("stage_router route {route_name}: {error}")) })?; + if let Some(target_modalities) = target_modalities { + algorithm = algorithm.with_target_modalities(target_modalities); + } Ok(Arc::new(algorithm)) } } @@ -1017,6 +1207,10 @@ fn default_classifier_max_output_tokens() -> u64 { TaskClassifierConfig::default().max_output_tokens } +const fn enabled_by_default() -> bool { + true +} + /// Keys each configured system prompt by the target it belongs to. fn tier_prompts( capable: &str, @@ -1066,6 +1260,30 @@ fn validate_value(label: &str, value: &str) -> ServerResult<()> { Ok(()) } +/// Rejects declarations that cannot describe a meaningful capability set. +fn validate_input_modalities( + target_name: &str, + modalities: Option<&[InputModality]>, +) -> ServerResult<()> { + let Some(modalities) = modalities else { + return Ok(()); + }; + if modalities.is_empty() { + return Err(ServerError::new(format!( + "target {target_name} input_modalities must not be empty" + ))); + } + let mut unique = BTreeSet::new(); + for modality in modalities { + if !unique.insert(*modality) { + return Err(ServerError::new(format!( + "target {target_name} input_modalities contains duplicate {modality}" + ))); + } + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -1128,6 +1346,7 @@ target = "weak" } } + // The bridge is opt-in and cannot silently no-op on another client format. #[test] fn builds_all_supported_algorithm_types() -> ServerResult<()> { let state = server_state_from_toml(VALID_CONFIG)?; @@ -1144,6 +1363,149 @@ target = "weak" Ok(()) } + #[test] + fn accepts_complete_target_input_modality_declarations() -> ServerResult<()> { + let configured = VALID_CONFIG + .replace( + "llm_client = \"primary\"\n\n[targets.strong]", + "llm_client = \"primary\"\ninput_modalities = [\"text\"]\n\n[targets.strong]", + ) + .replace( + "llm_client = \"responses\"\n\n[targets.weak]", + "llm_client = \"responses\"\ninput_modalities = [\"text\", \"image\", \"audio\", \"video\", \"file\"]\n\n[targets.weak]", + ) + .replace( + "llm_client = \"anthropic\"\n\n[routes.noop]", + "llm_client = \"anthropic\"\ninput_modalities = [\"text\", \"image\"]\n\n[routes.noop]", + ); + + server_state_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn rejects_invalid_target_input_modality_declarations() { + let cases = [ + ( + VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ninput_modalities = []", + ), + "input_modalities must not be empty", + ), + ( + VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ninput_modalities = [\"text\", \"text\"]", + ), + "input_modalities contains duplicate text", + ), + ( + VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ninput_modalities = [\"telepathy\"]", + ), + "unknown variant `telepathy`", + ), + ]; + + for (configured, expected) in cases { + let error = error_message(&configured); + assert!(error.contains(expected), "unexpected error: {error}"); + } + } + + #[test] + fn rejects_partial_route_modality_declarations() { + let configured = VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ninput_modalities = [\"text\", \"image\"]", + ); + + assert!( + error_message(&configured) + .contains("must declare input_modalities for every completion target or none") + ); + } + + #[test] + fn duplicate_model_ids_require_matching_route_modalities() { + const DUPLICATE_MODEL_CONFIG: &str = r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[targets.first] +id = "shared/model" +llm_client = "primary" +input_modalities = ["text", "image"] + +[targets.second] +id = "shared/model" +llm_client = "primary" +input_modalities = ["image", "text"] + +[routes.shared] +id = "switchyard/shared" +type = "random" +targets = ["first", "second"] +"#; + + // Repeated declarations for one model id may differ in ordering, but not content. + let config: ServerConfig = + toml::from_str(DUPLICATE_MODEL_CONFIG).expect("test config should parse"); + let route = config + .routes + .get("shared") + .expect("test route should exist"); + let (target_modalities, advertised) = config + .route_modalities("shared", route) + .expect("matching declarations should be accepted"); + assert_eq!( + target_modalities + .expect("target modalities should be declared") + .get(&ModelId::from("shared/model")), + Some(&BTreeSet::from( + [InputModality::Text, InputModality::Image,] + )) + ); + assert_eq!(advertised, [InputModality::Text, InputModality::Image]); + + let conflicting = DUPLICATE_MODEL_CONFIG.replace( + "input_modalities = [\"image\", \"text\"]", + "input_modalities = [\"text\"]", + ); + assert!( + error_message(&conflicting).contains( + "route shared maps model id shared/model to conflicting input_modalities" + ) + ); + } + + #[test] + fn judge_modality_declarations_require_text_but_are_optional() -> ServerResult<()> { + // Undeclared judge capabilities retain the legacy behavior. + server_state_from_toml(VALID_CONFIG)?; + + let valid = VALID_CONFIG.replace( + "llm_client = \"primary\"", + "llm_client = \"primary\"\ninput_modalities = [\"text\"]", + ); + server_state_from_toml(&valid)?; + + let invalid = VALID_CONFIG.replace( + "llm_client = \"primary\"", + "llm_client = \"primary\"\ninput_modalities = [\"image\"]", + ); + assert!( + error_message(&invalid) + .contains("judge target classifier for route classifier must include text") + ); + Ok(()) + } + #[test] fn an_escalation_table_switches_the_classifier_route_to_escalation() -> ServerResult<()> { // Present: the classifier target judges the weak tier's reply each turn instead of @@ -1348,6 +1710,13 @@ classifier_magic = true ), "message_hash_fallback requires session_affinity", ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nsession_affinity = true\nturn_affinity = true", + ), + "session_affinity and turn_affinity cannot both be enabled", + ), ( VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"), "unsupported schema_version 2", @@ -1471,12 +1840,23 @@ target = "azure" } #[test] - fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> { + fn accepts_turn_affinity() -> ServerResult<()> { + let configured = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nturn_affinity = true", + ); + server_state_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn target_request_settings_are_parsed_and_applied_to_its_backend() -> ServerResult<()> { let configured = VALID_CONFIG.replacen( "llm_client = \"primary\"", "llm_client = \"primary\"\n\ extra_body = { service_tier = \"priority\", \ - chat_template_kwargs = { enable_thinking = false } }", + chat_template_kwargs = { enable_thinking = false } }\n\ + reasoning_effort_override = \"max\"", 1, ); let config: ServerConfig = toml::from_str(&configured) @@ -1487,7 +1867,12 @@ target = "azure" let Some(client) = config.llm_clients.get("primary") else { return Err(ServerError::new("primary llm client is missing")); }; - let backend = build_backend("primary", client, &target.extra_body)?; + let backend = build_backend( + "primary", + client, + &target.extra_body, + target.reasoning_effort_override.as_deref(), + )?; assert_eq!( backend.extra_body().get("service_tier"), @@ -1500,6 +1885,106 @@ target = "azure" .and_then(|value| value.get("enable_thinking")), Some(&json!(false)) ); + assert_eq!(backend.reasoning_effort_override(), Some("max")); + Ok(()) + } + + #[test] + fn responses_custom_tool_bridge_is_explicit_and_format_scoped() -> ServerResult<()> { + let configured = VALID_CONFIG.replace( + "[llm_clients.responses]\nformat = \"openai_responses\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbridge_custom_tools = true", + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let Some(target) = config.targets.get("strong") else { + return Err(ServerError::new("strong target is missing")); + }; + let Some(client) = config.llm_clients.get("responses") else { + return Err(ServerError::new("responses llm client is missing")); + }; + let backend = build_backend( + "responses", + client, + &target.extra_body, + target.reasoning_effort_override.as_deref(), + )?; + assert!(backend.bridge_custom_tools()); + + let invalid = VALID_CONFIG.replace( + "[llm_clients.primary]\nformat = \"openai_chat\"", + "[llm_clients.primary]\nformat = \"openai_chat\"\nbridge_custom_tools = true", + ); + assert!( + error_message(&invalid) + .contains("bridge_custom_tools requires format openai_responses") + ); + Ok(()) + } + + #[test] + fn responses_tool_search_eager_loading_is_explicit_and_format_scoped() -> ServerResult<()> { + let configured = VALID_CONFIG.replace( + "[llm_clients.responses]\nformat = \"openai_responses\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\neager_load_tool_search = true", + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let Some(target) = config.targets.get("strong") else { + return Err(ServerError::new("strong target is missing")); + }; + let Some(client) = config.llm_clients.get("responses") else { + return Err(ServerError::new("responses llm client is missing")); + }; + let backend = build_backend( + "responses", + client, + &target.extra_body, + target.reasoning_effort_override.as_deref(), + )?; + assert!(backend.eager_load_tool_search()); + + let invalid = VALID_CONFIG.replace( + "[llm_clients.primary]\nformat = \"openai_chat\"", + "[llm_clients.primary]\nformat = \"openai_chat\"\neager_load_tool_search = true", + ); + assert!( + error_message(&invalid) + .contains("eager_load_tool_search requires format openai_responses") + ); + Ok(()) + } + + #[test] + fn xai_responses_compatibility_is_explicit_and_format_scoped() -> ServerResult<()> { + let configured = VALID_CONFIG.replace( + "[llm_clients.responses]\nformat = \"openai_responses\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nxai_responses_compatibility = true", + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let Some(target) = config.targets.get("strong") else { + return Err(ServerError::new("strong target is missing")); + }; + let Some(client) = config.llm_clients.get("responses") else { + return Err(ServerError::new("responses llm client is missing")); + }; + let backend = build_backend( + "responses", + client, + &target.extra_body, + target.reasoning_effort_override.as_deref(), + )?; + assert!(backend.xai_responses_compatibility()); + + let invalid = VALID_CONFIG.replace( + "[llm_clients.primary]\nformat = \"openai_chat\"", + "[llm_clients.primary]\nformat = \"openai_chat\"\nxai_responses_compatibility = true", + ); + assert!( + error_message(&invalid) + .contains("xai_responses_compatibility requires format openai_responses") + ); Ok(()) } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index fd02cd01b..9e4fe366e 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -36,8 +36,10 @@ use libsy::{Algorithm, LibsyError}; use parking_lot::Mutex; use serde::Deserialize; use serde_json::{Value, json}; -use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient}; -use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage}; +use switchyard_llm_client::{ + ClassifierContentObservation, ClientRouter, RunObservation, RunObserver, TranslatingLlmClient, +}; +use switchyard_protocol::{InputModality, LlmClientError, Metadata, ModelId, Request, Usage}; use tokio::net::{TcpListener, TcpSocket}; use tokio::task; use tracing::{Instrument, Level}; @@ -92,7 +94,7 @@ pub type ServerResult = std::result::Result; /// /// An unset capability is undeclared: it serializes as `null` in the OpenAI /// `data` entry, and the Codex entry falls back to a safe default for it. -#[derive(Clone, Copy, Default)] +#[derive(Clone)] struct ModelCapabilities { context_window: Option, tool_calling: Option, @@ -100,6 +102,23 @@ struct ModelCapabilities { // this, so a route opts in via config; undeclared routes advertise as // non-reasoning to Codex (fail closed). reasoning: Option, + // Whether the route can execute provider-hosted or translated web search. + web_search: Option, + // Always populated. Routes without target declarations use text as the + // safe discovery default. + input_modalities: Vec, +} + +impl Default for ModelCapabilities { + fn default() -> Self { + Self { + context_window: None, + tool_calling: None, + reasoning: None, + web_search: None, + input_modalities: vec![InputModality::Text], + } + } } /// A registered algorithm route and its server-owned endpoint metadata. @@ -161,6 +180,7 @@ pub struct ServerState { metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, + routing_log_classifier_content: bool, track_cache_eligibility: bool, } @@ -178,14 +198,46 @@ impl SharedRoutingLog { }) } - fn append( + fn append_usage( &self, context: routing_log::RoutingLogContext, model: &str, tier: Option<&str>, usage: &Usage, ) { - if let Err(error) = self.writer.lock().append(context, model, tier, usage) { + if let Err(error) = self.writer.lock().append_usage(context, model, tier, usage) { + tracing::warn!(path = %self.path.display(), %error, "routing log append failed"); + } + } + + fn append_event( + &self, + context: routing_log::RoutingLogContext, + event: &str, + model: &str, + elapsed: Duration, + outcome: Option<&str>, + ) { + if let Err(error) = self + .writer + .lock() + .append_event(context, event, model, elapsed, outcome) + { + tracing::warn!(path = %self.path.display(), %error, "routing log append failed"); + } + } + + fn append_classifier_content( + &self, + context: routing_log::RoutingLogContext, + content: &ClassifierContentObservation, + elapsed: Duration, + ) { + if let Err(error) = self + .writer + .lock() + .append_classifier_content(context, content, elapsed) + { tracing::warn!(path = %self.path.display(), %error, "routing log append failed"); } } @@ -260,6 +312,7 @@ impl ServerState { metrics, stats, routing_log: None, + routing_log_classifier_content: false, track_cache_eligibility: tracking_enabled_from_env(), }) } @@ -270,6 +323,15 @@ impl ServerState { Ok(self) } + /// Includes normalized classifier prompts, model reasoning, and verdicts in the routing log. + /// + /// The content may contain user-supplied secrets and should only be enabled for a protected + /// routing log. Transport headers and provider credentials are never part of this payload. + pub fn with_routing_log_classifier_content(mut self, enabled: bool) -> Self { + self.routing_log_classifier_content = enabled; + self + } + /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.routes.keys().map(ModelId::as_str) @@ -282,6 +344,13 @@ impl ServerState { .ok_or_else(|| ServerError::new(format!("unknown route model {model:?}"))) } + /// Returns the input modalities advertised for one route model. + pub fn input_modalities(&self, model: &str) -> ServerResult> { + self.route_for_model(model) + .map(|entry| entry.capabilities.input_modalities.clone()) + .ok_or_else(|| ServerError::new(format!("unknown route model {model:?}"))) + } + fn route_for_model(&self, model: &str) -> Option<&RouteEntry> { self.routes.get(model) } @@ -442,15 +511,45 @@ const CLASSIFIER_TIER: &str = "classifier"; /// Maps answer-call observations to backend stats, routing calls to classifier/judge /// stats, and records routing overhead once the algorithm run completes. /// -/// Successful classifier/judge calls are also appended to `classifier_log` when one is -/// configured, so per-session routing snapshots account for judge token overhead. Routed -/// calls stay off this path: the served call is logged with its terminal usage in -/// [`usage_metrics::observe`], which would make a second append here a double count. +/// Decisions and answer-call starts are appended immediately. Successful classifier/judge +/// calls also append usage so per-session snapshots account for judge overhead. Routed usage +/// stays on [`usage_metrics::observe`] to avoid double counting. fn stats_observer( stats: StatsAccumulator, - classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>, + routing_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>, + started: Instant, + log_classifier_content: bool, ) -> RunObserver { Arc::new(move |observation| match observation { + RunObservation::RoutingDecision(decision) => { + if let Some((log, context)) = routing_log.as_ref() { + log.append_event( + context.clone(), + "decision", + decision.selected_model_id().as_str(), + started.elapsed(), + None, + ); + } + } + RunObservation::LlmCallStarted(call) => { + if call.is_answer_call + && let Some((log, context)) = routing_log.as_ref() + { + log.append_event( + context.clone(), + "start", + call.selected_model.as_str(), + started.elapsed(), + None, + ); + } + } + RunObservation::ClassifierContent(content) => { + if log_classifier_content && let Some((log, context)) = routing_log.as_ref() { + log.append_classifier_content(context.clone(), &content, started.elapsed()); + } + } RunObservation::LlmCall(call) => { let latency_ms = call.duration.as_secs_f64() * 1_000.0; if call.is_answer_call { @@ -461,9 +560,9 @@ fn stats_observer( } } else if call.is_success { if let (Some((log, context)), Some(usage)) = - (classifier_log.as_ref(), call.usage.as_ref()) + (routing_log.as_ref(), call.usage.as_ref()) { - log.append( + log.append_usage( context.clone(), &call.selected_model, Some(CLASSIFIER_TIER), @@ -746,17 +845,56 @@ async fn handle_llm_request( Ok(resolved) => resolved, Err(response) => return response, }; + let requested_model = request + .llm_request + .model + .as_deref() + .unwrap_or("unknown") + .to_string(); + let lifecycle_log = state.routing_log.clone().zip(routing_log_context.clone()); + if let Some((log, context)) = lifecycle_log.as_ref() { + log.append_event( + context.clone(), + "request_start", + &requested_model, + started.0.elapsed(), + None, + ); + } let algorithm = Arc::clone(&route.algorithm); let client_router = route.target_clients.clone(); let observer = stats_observer( state.stats.clone(), state.routing_log.clone().zip(routing_log_context.clone()), + started.0, + state.routing_log_classifier_content, ); - let (trace, response) = - match switchyard_llm_client::run(algorithm, client_router, request, Some(observer)).await { - Ok(result) => result, - Err(error) => return algorithm_error(error), - }; + let observation_config = switchyard_llm_client::ObservationConfig { + classifier_content: state.routing_log_classifier_content, + }; + let (trace, response) = match switchyard_llm_client::run_with_observation_config( + algorithm, + client_router, + request, + Some(observer), + observation_config, + ) + .await + { + Ok(result) => result, + Err(error) => { + if let Some((log, context)) = lifecycle_log { + log.append_event( + context, + "completion", + &requested_model, + started.0.elapsed(), + Some("error"), + ); + } + return algorithm_error(error); + } + }; let decision = trace.last(); // The response carries the candidate that actually served it. Fall back to the routing // decision for algorithms that return a response without an offloaded model call. @@ -775,7 +913,7 @@ async fn handle_llm_request( started.0, state.stats, cache_eligible, - state.routing_log.zip(routing_log_context), + lifecycle_log, ) } else { response @@ -885,10 +1023,16 @@ fn sanitize_routing_header_value(value: &str) -> Option { } fn algorithm_error(error: LibsyError) -> Response { - let LibsyError::ClientCall { source, .. } = &error else { - return server_error(error.to_string()); - }; - client_error(source) + match &error { + LibsyError::NoCompatibleTargets { .. } => error_response( + StatusCode::BAD_REQUEST, + error.to_string(), + "invalid_request_error", + "unsupported_input_modalities", + ), + LibsyError::ClientCall { source, .. } => client_error(source), + _ => server_error(error.to_string()), + } } fn client_error(error: &LlmClientError) -> Response { @@ -1058,12 +1202,9 @@ fn error_response( } async fn models(State(state): State) -> Json { - Json(model_list_payload( - state - .routes - .iter() - .map(|(model, entry)| (model.as_str(), entry.capabilities)), - )) + Json(model_list_payload(state.routes.iter().map( + |(model, entry)| (model.as_str(), entry.capabilities.clone()), + ))) } async fn get_stats(State(state): State) -> Json { @@ -1152,11 +1293,11 @@ fn model_list_payload<'a>( let last_id = model_ids.last().copied(); json!({ "object": "list", - "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), + "data": entries.iter().map(|(model, caps)| model_entry_json(model, caps.clone())).collect::>(), "models": entries .iter() .enumerate() - .map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority)) + .map(|(priority, (model, caps))| codex_model_entry_json(model, caps.clone(), priority)) .collect::>(), "first_id": first_id, "last_id": last_id, @@ -1177,7 +1318,9 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { "capabilities": { "streaming": true, "tool_calling": capabilities.tool_calling, + "web_search": capabilities.web_search, "context_window": capabilities.context_window, + "input_modalities": capabilities.input_modalities, "supported_inbound_formats": [ "openai-chat-completions", "openai-responses", @@ -1242,8 +1385,8 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority "max_context_window": capabilities.context_window, "effective_context_window_percent": 95, "experimental_supported_tools": [], - "input_modalities": ["text"], - "supports_search_tool": false, + "input_modalities": capabilities.input_modalities, + "supports_search_tool": capabilities.web_search.unwrap_or(false), }) } @@ -1366,12 +1509,32 @@ mod tests { #[test] fn stats_observer_logs_judge_calls_to_the_routing_log() { let dir = tempfile::tempdir().expect("temp dir"); - let log = SharedRoutingLog::new(dir.path().join("routing.jsonl")).expect("routing log"); + let log_path = dir.path().join("routing.jsonl"); + let log = SharedRoutingLog::new(log_path.clone()).expect("routing log"); let mut headers = HeaderMap::new(); headers.insert("proxy_x_session_id", "session-1".parse().expect("header")); let metadata = metadata_from_headers(headers); let context = routing_log::RoutingLogContext::from_metadata(&metadata); - let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context))); + let observer = stats_observer( + StatsAccumulator::default(), + Some((log.clone(), context)), + Instant::now(), + true, + ); + + observer(RunObservation::ClassifierContent( + ClassifierContentObservation { + selected_model: ModelId::from("judge-model"), + request: switchyard_protocol::text_request( + Some("judge-model".to_string()), + "review code", + ), + reasoning: Some("This is substantive review work.".to_string()), + verdict: Some(r#"{"target":"luna"}"#.to_string()), + is_success: true, + duration: Duration::from_millis(3), + }, + )); let call = |model: &str, is_answer_call: bool| { RunObservation::LlmCall(LlmCallObservation { @@ -1398,6 +1561,23 @@ mod tests { assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100); assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7); assert!(snapshot["models"].get("routed-model").is_none()); + + let content_record = std::fs::read_to_string(log_path) + .expect("read routing log") + .lines() + .map(|line| serde_json::from_str::(line).expect("valid JSONL")) + .find(|record| record["event"] == "classifier_content") + .expect("classifier content record"); + assert_eq!(content_record["model"], "judge-model"); + assert_eq!( + content_record["classifier_request"]["messages"][0]["content"][0]["text"], + "review code" + ); + assert_eq!( + content_record["classifier_reasoning"], + "This is substantive review work." + ); + assert_eq!(content_record["classifier_verdict"], r#"{"target":"luna"}"#); } #[derive(Clone)] diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index 47ed65387..efbc9521d 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -8,10 +8,14 @@ use std::collections::BTreeMap; use std::fs; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; use std::time::SystemTime; use humantime::format_rfc3339_millis; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use switchyard_llm_client::ClassifierContentObservation; use switchyard_protocol::{Metadata, ModelId, Usage}; use crate::usage_metrics::token_usage; @@ -20,6 +24,7 @@ use crate::{ServerError, ServerResult}; const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; const TASK_HEADER: &str = "x-switchyard-intake-task"; const TRIAL_ID_HEADER: &str = "x-switchyard-trial-id"; +static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1); /// Append-only writer for one routing JSONL file. pub(crate) struct RoutingLog(fs::File); @@ -41,7 +46,7 @@ impl RoutingLog { Ok(Self(file)) } - pub(crate) fn append( + pub(crate) fn append_usage( &mut self, context: RoutingLogContext, model: &str, @@ -51,9 +56,12 @@ impl RoutingLog { let usage = token_usage(usage); let record = RoutingRecord { ts: format_rfc3339_millis(SystemTime::now()).to_string().into(), + event: "usage".into(), task: context.task.map(Cow::Owned), trial_id: context.trial_id.map(Cow::Owned), session_id: context.session_id.map(Cow::Owned), + request_id: Some(context.request_id.into()), + correlation_id: context.correlation_id.map(Cow::Owned), model: model.into(), tier: tier.unwrap_or("").into(), prompt_tokens: usage.prompt_tokens, @@ -62,7 +70,67 @@ impl RoutingLog { completion_tokens: usage.completion_tokens, reasoning_tokens: usage.reasoning_tokens, total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens), + ..RoutingRecord::default() }; + self.write_record(&record) + } + + /// Appends one request-lifecycle event without depending on provider token usage. + pub(crate) fn append_event( + &mut self, + context: RoutingLogContext, + event: &str, + model: &str, + elapsed: Duration, + outcome: Option<&str>, + ) -> std::io::Result<()> { + let record = RoutingRecord { + ts: format_rfc3339_millis(SystemTime::now()).to_string().into(), + event: event.into(), + task: context.task.map(Cow::Owned), + trial_id: context.trial_id.map(Cow::Owned), + session_id: context.session_id.map(Cow::Owned), + request_id: Some(context.request_id.into()), + correlation_id: context.correlation_id.map(Cow::Owned), + model: model.into(), + outcome: outcome.unwrap_or("").into(), + elapsed_ms: Some(elapsed.as_secs_f64() * 1_000.0), + ..RoutingRecord::default() + }; + self.write_record(&record) + } + + /// Appends the exact normalized classifier request and model-produced response content. + pub(crate) fn append_classifier_content( + &mut self, + context: RoutingLogContext, + content: &ClassifierContentObservation, + elapsed: Duration, + ) -> std::io::Result<()> { + let classifier_request = + serde_json::to_value(&content.request).map_err(std::io::Error::other)?; + let record = RoutingRecord { + ts: format_rfc3339_millis(SystemTime::now()).to_string().into(), + event: "classifier_content".into(), + task: context.task.map(Cow::Owned), + trial_id: context.trial_id.map(Cow::Owned), + session_id: context.session_id.map(Cow::Owned), + request_id: Some(context.request_id.into()), + correlation_id: context.correlation_id.map(Cow::Owned), + model: content.selected_model.as_str().into(), + tier: "classifier".into(), + outcome: if content.is_success { "ok" } else { "error" }.into(), + elapsed_ms: Some(elapsed.as_secs_f64() * 1_000.0), + call_duration_ms: Some(content.duration.as_secs_f64() * 1_000.0), + classifier_request: Some(classifier_request), + classifier_reasoning: content.reasoning.as_deref().map(Cow::Borrowed), + classifier_verdict: content.verdict.as_deref().map(Cow::Borrowed), + ..RoutingRecord::default() + }; + self.write_record(&record) + } + + fn write_record(&mut self, record: &RoutingRecord<'_>) -> std::io::Result<()> { let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?; line.push(b'\n'); @@ -101,6 +169,8 @@ pub(crate) struct RoutingLogContext { task: Option, trial_id: Option, session_id: Option, + request_id: String, + correlation_id: Option, } impl RoutingLogContext { @@ -119,6 +189,12 @@ impl RoutingLogContext { .and_then(|headers| nonempty_header(headers, LEGACY_SESSION_ID_HEADER)) .map(str::to_string) }), + request_id: format!( + "local-{}-{}", + std::process::id(), + NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed) + ), + correlation_id: metadata.correlation_id.clone(), } } } @@ -130,14 +206,27 @@ impl RoutingLogContext { #[serde(default)] struct RoutingRecord<'a> { ts: Cow<'a, str>, + event: Cow<'a, str>, #[serde(borrow)] task: Option>, #[serde(borrow)] trial_id: Option>, #[serde(borrow)] session_id: Option>, + #[serde(borrow)] + request_id: Option>, + #[serde(borrow)] + correlation_id: Option>, model: Cow<'a, str>, tier: Cow<'a, str>, + outcome: Cow<'a, str>, + elapsed_ms: Option, + call_duration_ms: Option, + classifier_request: Option, + #[serde(borrow)] + classifier_reasoning: Option>, + #[serde(borrow)] + classifier_verdict: Option>, prompt_tokens: u64, cached_tokens: u64, cache_creation_tokens: u64, @@ -184,6 +273,9 @@ impl SessionStatsSnapshot { if record.session_id.as_deref() != Some(session_id) { return; } + if !matches!(record.event.as_ref(), "" | "usage") { + return; + } let model = match record.model.as_ref() { "" => "unknown", model => model, @@ -244,8 +336,12 @@ mod tests { fs::write( &path, concat!( + r#"{"event":"start","session_id":"a","model":"route","elapsed_ms":0.1}"#, + "\n", r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2}"#, "\n", + r#"{"event":"completion","session_id":"a","model":"m1","outcome":"ok","elapsed_ms":1.0}"#, + "\n", r#"{"session_id":"b","model":"m1","prompt_tokens":99,"completion_tokens":99}"#, "\n", "not json\n", diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index 1a3e10cba..099043fba 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -27,19 +27,21 @@ pub(crate) fn observe( metadata, } = response; let model = model.to_string(); + let mut lifecycle = RoutingLifecycle::new(routing_log, model.clone(), started); let llm_response = match llm_response { LlmResponse::Agg(agg) => { + lifecycle.ttfb(); record_terminal(&stats, &agg.usage, &model, started, cache_eligible); - if let Some((log, context)) = routing_log { - log.append(context, &model, None, &agg.usage); - } + lifecycle.append_usage(&agg.usage); + lifecycle.complete("ok"); LlmResponse::Agg(agg) } LlmResponse::Stream(mut stream) => { let wrapped = async_stream::stream! { let mut latest_usage = None; while let Some(item) = stream.next().await { + lifecycle.ttfb(); let failed = match &item { Err(_) => true, Ok(event) => event.normalized().iter().any(|chunk| { @@ -59,6 +61,7 @@ pub(crate) fn observe( } if failed { record_stream_error(&stats, &model); + lifecycle.complete("error"); } yield item; if failed { @@ -67,9 +70,8 @@ pub(crate) fn observe( } let usage = latest_usage.unwrap_or_default(); record_terminal(&stats, &usage, &model, started, cache_eligible); - if let Some((log, context)) = routing_log { - log.append(context, &model, None, &usage); - } + lifecycle.append_usage(&usage); + lifecycle.complete("ok"); }; LlmResponse::Stream(Box::pin(wrapped)) } @@ -81,6 +83,77 @@ pub(crate) fn observe( } } +/// Durable lifecycle events for one routed response, including cancellation on drop. +struct RoutingLifecycle { + routing_log: Option<(SharedRoutingLog, RoutingLogContext)>, + model: String, + started: Instant, + saw_first_byte: bool, + completed: bool, +} + +impl RoutingLifecycle { + fn new( + routing_log: Option<(SharedRoutingLog, RoutingLogContext)>, + model: String, + started: Instant, + ) -> Self { + Self { + routing_log, + model, + started, + saw_first_byte: false, + completed: false, + } + } + + fn ttfb(&mut self) { + if self.saw_first_byte { + return; + } + self.saw_first_byte = true; + if let Some((log, context)) = self.routing_log.as_ref() { + log.append_event( + context.clone(), + "ttfb", + &self.model, + self.started.elapsed(), + None, + ); + } + } + + fn append_usage(&self, usage: &Usage) { + if let Some((log, context)) = self.routing_log.as_ref() { + log.append_usage(context.clone(), &self.model, None, usage); + } + } + + fn complete(&mut self, outcome: &str) { + if self.completed { + return; + } + self.completed = true; + if let Some((log, context)) = self.routing_log.as_ref() { + log.append_event( + context.clone(), + "completion", + &self.model, + self.started.elapsed(), + Some(outcome), + ); + } + } +} + +impl Drop for RoutingLifecycle { + fn drop(&mut self) { + if !self.completed { + self.complete("cancelled"); + } + } +} + // Records a terminal stream failure after the routed call was already counted. fn record_stream_error(stats: &StatsAccumulator, model: &str) { stats.record_stream_error(model); diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 1cd87dca7..8d9b2d88c 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -113,7 +113,10 @@ async fn upstream_chat( let model = body["model"].as_str().unwrap_or("unknown").to_string(); let prompt = body["messages"][0]["content"].as_str().unwrap_or(""); - if (model == "model/weak" && prompt == "unavailable") || prompt == "all-unavailable" { + if (model == "model/weak" && prompt == "unavailable") + || (model == "model/premium" && prompt == "premium-unavailable") + || prompt == "all-unavailable" + { return ( StatusCode::SERVICE_UNAVAILABLE, Json(json!({"error": {"message": "upstream is unavailable"}})), @@ -340,6 +343,10 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult>(); + assert_eq!(outage_models.first(), Some(&"model/classifier")); + assert!(outage_models.len() > 1); + assert!( + outage_models[1..] + .iter() + .all(|model| *model == "model/premium") + ); + drop(calls); + let calls = upstream.calls.lock().await; let judge_call = calls .iter() @@ -1160,6 +1194,108 @@ selector = "/decision/target" Ok(()) } +#[tokio::test] +async fn custom_classifier_can_hide_images_from_a_text_only_judge() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" +input_modalities = ["text"] + +[targets.strong] +id = "model/strong" +llm_client = "upstream" +input_modalities = ["text", "image"] + +[targets.premium] +id = "model/premium" +llm_client = "upstream" +input_modalities = ["text", "image"] + +[routes.custom] +id = "switchyard/custom-image" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["strong", "premium"] +default_target = "strong" +judge_text_only = true +prompt = "CUSTOM IMAGE ROUTER" +response_schema = ''' +{{ + "type": "object", + "properties": {{ + "decision": {{ + "type": "object", + "properties": {{ + "target": {{"type": "string", "enum": ["strong", "premium"]}} + }}, + "required": ["target"], + "additionalProperties": false + }} + }}, + "required": ["decision"], + "additionalProperties": false +}} +''' + +[routes.custom.policy] +type = "target_selector" +selector = "/decision/target" +"#, + base_url = upstream.base_url, + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/custom-image", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "Inspect this image carefully."}, + {"type": "image_url", "image_url": {"url": "https://example.test/private.png"}} + ]}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/premium") + ); + + let calls = upstream.calls.lock().await; + assert_eq!(calls.len(), 2); + let judge = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("classifier was not called")?; + let completion = calls + .iter() + .find(|call| call["model"] == "model/premium") + .ok_or("selected completion target was not called")?; + let judge_wire = serde_json::to_string(judge)?; + assert!(judge_wire.contains("[image input omitted from text-only classifier]")); + assert!(!judge_wire.contains("private.png")); + assert!(!judge_wire.contains("image_url")); + assert!(serde_json::to_string(completion)?.contains("private.png")); + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; @@ -1282,6 +1418,127 @@ prompt = "CUSTOM STAGE" Ok(()) } +#[tokio::test] +async fn classifier_and_stage_routes_skip_judges_when_modalities_force_a_target() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" +max_retries = 0 + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" +input_modalities = ["text"] + +[targets.text] +id = "model/text" +llm_client = "upstream" +input_modalities = ["text"] + +[targets.vision] +id = "model/vision" +llm_client = "upstream" +input_modalities = ["text", "image"] + +[routes.capability] +id = "switchyard/capability-modalities" +type = "llm_classifier" +mode = "capability" +classifier_target = "classifier" +strong_target = "vision" +weak_target = "text" +base_threshold = 0.5 + +[routes.escalation] +id = "switchyard/escalation-modalities" +type = "llm_classifier" +mode = "escalation" +classifier_target = "classifier" +strong_target = "vision" +weak_target = "text" +escalation = {{ confirmations = 1 }} + +[routes.custom] +id = "switchyard/custom-modalities" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["text", "vision"] +default_target = "text" +prompt = "Select a target" +response_schema = ''' +{{ + "type": "object", + "properties": {{"target": {{"type": "string", "enum": ["text", "vision"]}}}}, + "required": ["target"], + "additionalProperties": false +}} +''' + +[routes.custom.policy] +type = "target_selector" +selector = "/target" + +[routes.stage] +id = "switchyard/stage-modalities" +type = "stage_router" +capable_target = "vision" +efficient_target = "text" +picker = "efficient_first" +confidence_threshold = 1.0 + +[routes.stage.classifier] +target = "classifier" +base_threshold = 0.5 +"#, + base_url = upstream.base_url, + ))?; + let app = build_switchyard_router(state); + + for route in [ + "switchyard/capability-modalities", + "switchyard/escalation-modalities", + "switchyard/custom-modalities", + "switchyard/stage-modalities", + ] { + upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": route, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "describe this"}, + {"type": "image_url", "image_url": {"url": "https://example.test/image.png"}} + ]}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK, "{route}"); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/vision"), + "{route}" + ); + assert_eq!( + upstream.models().await, + ["model/vision"], + "{route} made an unnecessary classifier or efficient-tier call" + ); + } + Ok(()) +} + #[tokio::test] async fn stage_classifier_can_request_json_object_output() -> TestResult { let upstream = MockUpstream::start().await?; @@ -1671,12 +1928,28 @@ base_url = "https://example.test/v1" id = "nvidia/deepseek-ai/deepseek-v4-pro" llm_client = "primary" +[targets.text] +id = "text/model" +llm_client = "primary" +input_modalities = ["text"] + +[targets.multimodal] +id = "multimodal/model" +llm_client = "primary" +input_modalities = ["audio", "image", "text"] + +[targets.judge] +id = "judge/model" +llm_client = "primary" +input_modalities = ["text", "file"] + [routes.declared] id = "declared" type = "passthrough" target = "shared" context_window = 1000000 tool_calling = true +web_search = true [routes.restricted] id = "restricted" @@ -1695,6 +1968,19 @@ reasoning = true id = "undeclared" type = "passthrough" target = "shared" + +[routes.multimodal] +id = "multimodal" +type = "random" +targets = ["text", "multimodal"] + +[routes.judged] +id = "judged" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "multimodal" +weak_target = "text" +base_threshold = 0.5 "#; let app = build_switchyard_router(load_test_config(CONFIG)?); let models = send(&app, "GET", "/v1/models", None).await?; @@ -1708,10 +1994,24 @@ target = "shared" assert_eq!(capabilities["declared"]["context_window"], json!(1_000_000)); assert_eq!(capabilities["declared"]["tool_calling"], json!(true)); + assert_eq!(capabilities["declared"]["web_search"], json!(true)); assert_eq!(capabilities["restricted"]["context_window"], json!(262_000)); assert_eq!(capabilities["restricted"]["tool_calling"], json!(false)); assert_eq!(capabilities["undeclared"]["context_window"], json!(null)); assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null)); + assert_eq!(capabilities["undeclared"]["web_search"], json!(null)); + assert_eq!( + capabilities["multimodal"]["input_modalities"], + json!(["text", "image", "audio"]) + ); + assert_eq!( + capabilities["judged"]["input_modalities"], + json!(["text", "image", "audio"]) + ); + assert_eq!( + capabilities["undeclared"]["input_modalities"], + json!(["text"]) + ); let codex_models = body["models"].as_array().cloned().unwrap_or_default(); let codex_metadata = codex_models @@ -1720,7 +2020,7 @@ target = "shared" .collect::>(); // This checks the shape the server emits. That Codex 0.144.5 actually decodes it // (context_window: null included) is verified by a live Codex run in SWITCH-1225. - assert_eq!(codex_metadata.len(), 4); + assert_eq!(codex_metadata.len(), 6); assert_eq!( codex_metadata["declared"]["context_window"], json!(1_000_000) @@ -1734,10 +2034,26 @@ target = "shared" assert_eq!(codex_metadata["declared"]["visibility"], "list"); assert_eq!(codex_metadata["declared"]["supported_in_api"], json!(true)); assert_eq!(codex_metadata["declared"]["web_search_tool_type"], "text"); + assert_eq!( + codex_metadata["declared"]["supports_search_tool"], + json!(true) + ); + assert_eq!( + codex_metadata["undeclared"]["supports_search_tool"], + json!(false) + ); assert_eq!( codex_metadata["declared"]["input_modalities"], json!(["text"]) ); + assert_eq!( + codex_metadata["multimodal"]["input_modalities"], + json!(["text", "image", "audio"]) + ); + assert_eq!( + codex_metadata["judged"]["input_modalities"], + json!(["text", "image", "audio"]) + ); assert_eq!( codex_metadata["declared"]["truncation_policy"], json!({"mode": "tokens", "limit": 10_000}) @@ -1874,6 +2190,197 @@ async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestRes Ok(()) } +#[tokio::test] +async fn image_requests_route_only_to_vision_targets_and_preserve_the_image() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.mock] +format = "openai_chat" +base_url = "{base_url}" +max_retries = 0 + +[targets.text] +id = "model/text" +llm_client = "mock" +input_modalities = ["text"] + +[targets.vision] +id = "model/vision" +llm_client = "mock" +input_modalities = ["text", "image"] + +[routes.multimodal] +id = "switchyard/multimodal" +type = "random" +targets = ["text", "vision"] +weights = [1, 1] +"#, + base_url = upstream.base_url, + ))?; + let app = build_switchyard_router(state); + let cases = [ + ( + "/v1/chat/completions", + json!({ + "model": "switchyard/multimodal", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + {"type": "image_url", "image_url": {"url": "https://example.test/image.png"}} + ] + }] + }), + ), + ( + "/v1/responses", + json!({ + "model": "switchyard/multimodal", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "describe this"}, + {"type": "input_image", "image_url": {"url": "https://example.test/image.png"}} + ] + }] + }), + ), + ( + "/v1/messages", + json!({ + "model": "switchyard/multimodal", + "max_tokens": 16, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + {"type": "image", "source": { + "type": "base64", + "media_type": "image/png", + "data": "AAAA" + }} + ] + }] + }), + ), + ]; + + for (path, body) in cases { + let response = send(&app, "POST", path, Some(body)).await?; + assert_eq!(response.status, StatusCode::OK, "{path}"); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/vision"), + "{path}" + ); + } + + let calls = upstream.calls.lock().await; + assert_eq!(calls.len(), 3); + for call in calls.iter() { + assert_eq!(call["model"], "model/vision"); + let retained_image = call["messages"] + .as_array() + .into_iter() + .flatten() + .filter_map(|message| message["content"].as_array()) + .flatten() + .any(|block| block["type"] == "image_url"); + assert!( + retained_image, + "image was removed from upstream call: {call}" + ); + } + Ok(()) +} + +#[tokio::test] +async fn unsupported_images_return_provider_errors_without_upstream_calls() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.mock] +format = "openai_chat" +base_url = "{base_url}" +max_retries = 0 + +[targets.text] +id = "model/text" +llm_client = "mock" +input_modalities = ["text"] + +[routes.text] +id = "switchyard/text" +type = "passthrough" +target = "text" +"#, + base_url = upstream.base_url, + ))?; + let app = build_switchyard_router(state); + let cases = [ + ( + "/v1/chat/completions", + json!({ + "model": "switchyard/text", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://example.test/image.png"}} + ]}] + }), + false, + ), + ( + "/v1/responses", + json!({ + "model": "switchyard/text", + "input": [{"type": "message", "role": "user", "content": [ + {"type": "input_image", "image_url": {"url": "https://example.test/image.png"}} + ]}] + }), + false, + ), + ( + "/v1/messages", + json!({ + "model": "switchyard/text", + "max_tokens": 16, + "messages": [{"role": "user", "content": [{"type": "image", "source": { + "type": "base64", "media_type": "image/png", "data": "AAAA" + }}]}] + }), + true, + ), + ]; + + for (path, body, anthropic) in cases { + let response = send(&app, "POST", path, Some(body)).await?; + assert_eq!(response.status, StatusCode::BAD_REQUEST, "{path}"); + let body = response.json()?; + if anthropic { + assert_eq!(body["type"], "error"); + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .is_some_and(|message| { message.contains("no compatible targets") }) + ); + } else { + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body["error"]["code"], "unsupported_input_modalities"); + } + } + assert!(upstream.calls.lock().await.is_empty()); + Ok(()) +} + // Normalized metadata is authoritative when both ID forms are present; // legacy-only callers remain supported for backward compatibility. #[tokio::test] @@ -1960,8 +2467,9 @@ async fn routing_log_prefers_canonical_and_preserves_legacy_fallback() -> TestRe async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TestResult { let upstream = MockUpstream::start().await?; let temp_dir = tempfile::tempdir()?; + let log_path = temp_dir.path().join("routing.jsonl"); let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? - .with_routing_log(temp_dir.path().join("routing.jsonl"))?; + .with_routing_log(&log_path)?; let app = build_switchyard_router(state); // `send_with_headers` collects the response body, so the stream wrapper reaches @@ -1995,6 +2503,72 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> T assert_eq!(stats["total_cached_tokens"], 7); assert_eq!(stats["total_cache_creation_tokens"], 2); assert_eq!(stats["total_completion_tokens"], 5); + + let records = std::fs::read_to_string(log_path)? + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + assert_eq!( + records + .iter() + .map(|record| record["event"].as_str().unwrap_or("")) + .collect::>(), + [ + "request_start", + "decision", + "start", + "ttfb", + "usage", + "completion" + ] + ); + assert!(records.iter().all(|record| { + record["request_id"].as_str() == records[0]["request_id"].as_str() + && record["elapsed_ms"].as_f64().is_some() != (record["event"] == "usage") + })); + assert_eq!( + records.last().and_then(|record| record["outcome"].as_str()), + Some("ok") + ); + Ok(()) +} + +#[tokio::test] +async fn routing_log_records_stream_cancellation_without_usage() -> TestResult { + let upstream = MockUpstream::start().await?; + let temp_dir = tempfile::tempdir()?; + let log_path = temp_dir.path().join("routing.jsonl"); + let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? + .with_routing_log(&log_path)?; + let app = build_switchyard_router(state); + + let request = HttpRequest::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .header("x-switchyard-session-id", "cancelled-session") + .body(Body::from(serde_json::to_vec(&json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "hello"}], + "stream": true + }))?))?; + let response = app.oneshot(request).await?; + assert_eq!(response.status(), StatusCode::OK); + drop(response); + + let records = std::fs::read_to_string(log_path)? + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + assert_eq!( + records + .iter() + .map(|record| record["event"].as_str().unwrap_or("")) + .collect::>(), + ["request_start", "decision", "start", "completion"] + ); + assert_eq!(records[3]["outcome"], "cancelled"); + assert!(records.iter().all(|record| record["event"] != "usage")); Ok(()) } @@ -2061,8 +2635,12 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted .lines() .map(serde_json::from_str::) .collect::, _>>()?; - assert_eq!(records.len(), 3); - assert!(records.iter().all(|record| { + let usage_records = records + .iter() + .filter(|record| record["event"] == "usage") + .collect::>(); + assert_eq!(usage_records.len(), 3); + assert!(usage_records.iter().all(|record| { record["model"] == "model/strong" && record.get("fallback_reason").is_none() })); diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index fe5573039..3e2d3e6c8 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -803,6 +803,43 @@ fn encode_one_anthropic_response_block(block: &ContentBlock) -> Vec { } } +// Splits an image data URL into the media type and base64 payload Anthropic expects. +fn base64_image_data_url(url: &str) -> Option<(&str, &str)> { + let (metadata, data) = url.strip_prefix("data:")?.split_once(',')?; + let mut parts = metadata.split(';'); + let media_type = parts.next()?; + (media_type.starts_with("image/") && parts.any(|part| part.eq_ignore_ascii_case("base64"))) + .then_some((media_type, data)) +} + +// Encodes a normalized image source without its surrounding Anthropic image block. +fn encode_anthropic_image_source(source: &ImageSource) -> Value { + match source { + ImageSource::Url { url, .. } => match base64_image_data_url(url) { + Some((media_type, data)) => json!({ + "type": "base64", + "media_type": media_type, + "data": data, + }), + None => json!({"type": "url", "url": url}), + }, + ImageSource::Base64 { media_type, data } => json!({ + "type": "base64", + "media_type": media_type.clone().unwrap_or_else(|| "image/png".to_string()), + "data": data, + }), + ImageSource::Raw(raw) => raw.clone(), + } +} + +// Raw sources already retain the complete Anthropic block and must not be wrapped again. +fn encode_anthropic_image_block(source: &ImageSource) -> Value { + match source { + ImageSource::Raw(raw) => raw.clone(), + source => json!({"type": "image", "source": encode_anthropic_image_source(source)}), + } +} + // Encodes a single normalized content block into Anthropic block JSON. fn encode_one_anthropic_block(block: &ContentBlock) -> Vec { match block { @@ -847,20 +884,7 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec { "content": content, })] } - ContentBlock::Image { source } => vec![match source { - ImageSource::Url { url, .. } => { - json!({"type": "image", "source": {"type": "url", "url": url}}) - } - ImageSource::Base64 { media_type, data } => json!({ - "type": "image", - "source": { - "type": "base64", - "media_type": media_type.clone().unwrap_or_else(|| "image/png".to_string()), - "data": data, - }, - }), - ImageSource::Raw(raw) => raw.clone(), - }], + ContentBlock::Image { source } => vec![encode_anthropic_image_block(source)], ContentBlock::File { source } => vec![match source { FileSource::FileId(file_id) => { json!({"type": "document", "source": {"type": "file", "file_id": file_id}}) diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 32d0de1a2..f89e35508 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -480,6 +480,16 @@ pub(crate) fn decode_openai_content( source: decode_file_source(block), }); } + Some("audio") | Some("audio_url") | Some("input_audio") => { + content.push(ContentBlock::Audio { + source: decode_media_source(block, "audio"), + }); + } + Some("video") | Some("video_url") | Some("input_video") => { + content.push(ContentBlock::Video { + source: decode_media_source(block, "video"), + }); + } _ => content.push(ContentBlock::Unknown { provider: provider.into(), raw: Value::Object(block.clone()), @@ -561,6 +571,76 @@ pub(crate) fn decode_file_source(block: &Map) -> FileSource { FileSource::Raw(Value::Object(block.clone())) } +/// Decodes common OpenAI audio and video block shapes into normalized media sources. +pub(crate) fn decode_media_source(block: &Map, kind: &str) -> MediaSource { + let url_key = format!("{kind}_url"); + if let Some(value) = block.get(&url_key) { + if let Some(url) = value.as_str() { + return MediaSource::Url { + url: url.to_string(), + media_type: media_type(block, kind), + }; + } + if let Some(payload) = value.as_object() + && let Some(url) = payload.get("url").and_then(Value::as_str) + { + return MediaSource::Url { + url: url.to_string(), + media_type: media_type(payload, kind).or_else(|| media_type(block, kind)), + }; + } + } + + let input_key = format!("input_{kind}"); + for key in [input_key.as_str(), kind] { + let Some(value) = block.get(key) else { + continue; + }; + if let Some(data) = value.as_str() { + return MediaSource::Base64 { + media_type: media_type(block, kind), + data: data.to_string(), + }; + } + if let Some(payload) = value.as_object() { + if let Some(url) = payload.get("url").and_then(Value::as_str) { + return MediaSource::Url { + url: url.to_string(), + media_type: media_type(payload, kind).or_else(|| media_type(block, kind)), + }; + } + if let Some(data) = payload.get("data").and_then(Value::as_str) { + return MediaSource::Base64 { + media_type: media_type(payload, kind).or_else(|| media_type(block, kind)), + data: data.to_string(), + }; + } + } + } + + if let Some(data) = block.get("data").and_then(Value::as_str) { + return MediaSource::Base64 { + media_type: media_type(block, kind), + data: data.to_string(), + }; + } + MediaSource::Raw(Value::Object(block.clone())) +} + +fn media_type(object: &Map, kind: &str) -> Option { + object + .get("media_type") + .or_else(|| object.get("format")) + .and_then(Value::as_str) + .map(|value| { + if value.contains('/') { + value.to_string() + } else { + format!("{kind}/{value}") + } + }) +} + /// Decodes one OpenAI tool call into a normalized tool call. pub(crate) fn decode_openai_tool_call( tool_call: &Value, @@ -1140,6 +1220,7 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage { .get("output_tokens_details") .and_then(|details| details.get("reasoning_tokens")) }) + .or_else(|| value.get("reasoning_tokens")) .and_then(Value::as_u64), } } diff --git a/crates/switchyard-translation/src/codecs/openai_chat/mod.rs b/crates/switchyard-translation/src/codecs/openai_chat/mod.rs index 766f312c6..159e00b1d 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/mod.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/mod.rs @@ -9,4 +9,4 @@ mod stream; pub use buffered::OpenAiChatCodec; pub use stream::OpenAiChatStreamCodec; -pub(crate) use buffered::{decode_file_source, decode_image_source}; +pub(crate) use buffered::{decode_file_source, decode_image_source, decode_media_source}; diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index b4c6273b1..218784941 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -284,6 +284,7 @@ fn openai_usage(usage: &Map) -> Usage { .get("output_tokens_details") .and_then(|details| details.get("reasoning_tokens")) }) + .or_else(|| usage.get("reasoning_tokens")) .and_then(Value::as_u64), } } diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 025dbe35f..3d6f0fa5c 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -10,7 +10,7 @@ use serde_json::{Map, Value, json}; use crate::codecs::common::{ is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks, }; -use crate::codecs::openai_chat::{decode_file_source, decode_image_source}; +use crate::codecs::openai_chat::{decode_file_source, decode_image_source, decode_media_source}; use crate::codecs::{ DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec, }; @@ -327,7 +327,14 @@ fn decode_responses_input( )?; continue; }; - match item.get("type").and_then(Value::as_str) { + // Responses input messages may omit their otherwise constant `type`. + let item_type = match item.get("type") { + None if item.contains_key("role") && item.contains_key("content") => { + Some("message") + } + value => value.and_then(Value::as_str), + }; + match item_type { Some("message") => { let role = request_role_from_responses( item.get("role").and_then(Value::as_str), @@ -416,10 +423,18 @@ fn decode_responses_input( .and_then(Value::as_str) .unwrap_or_default() .to_string(); - let output_text = item.get("output").map(json_string).unwrap_or_default(); + let content = match item.get("output") { + Some(output @ Value::Array(_)) => decode_responses_content(output), + Some(output) => vec![ContentBlock::Text { + text: json_string(output), + }], + None => vec![ContentBlock::Text { + text: String::new(), + }], + }; pending_tool_outputs.push(ToolResult { tool_call_id, - content: vec![ContentBlock::Text { text: output_text }], + content, is_error: None, }); } @@ -659,6 +674,16 @@ fn decode_responses_content(value: &Value) -> Vec { Some("input_file") => out.push(ContentBlock::File { source: decode_file_source(block), }), + Some("audio") | Some("audio_url") | Some("input_audio") => { + out.push(ContentBlock::Audio { + source: decode_media_source(block, "audio"), + }); + } + Some("video") | Some("video_url") | Some("input_video") => { + out.push(ContentBlock::Video { + source: decode_media_source(block, "video"), + }); + } _ => out.push(ContentBlock::Unknown { provider: WireFormat::OpenAiResponses.into(), raw: Value::Object(block.clone()), diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 8a7d324ec..6e00a76a2 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -224,6 +224,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec { "output_index": output_index, "item": { "type": "message", + "id": format!("msg_{output_index}"), "role": "assistant", "status": status, "content": [{"type": "output_text", "text": state.response_text}], @@ -265,6 +266,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec { output_index, json!({ "type": "message", + "id": format!("msg_{output_index}"), "role": "assistant", "status": status, "content": [{"type": "output_text", "text": state.response_text}], diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index e14d38e9b..b15af4e8f 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -6,11 +6,97 @@ use pretty_assertions::assert_eq; use serde_json::{Value, json}; use switchyard_translation::{ - LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat, + InputModality, LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat, }; type TestResult = std::result::Result<(), Box>; +#[test] +fn openai_request_media_blocks_are_typed_for_modality_routing() -> TestResult { + let engine = TranslationEngine::default(); + let chat = engine.decode_request( + WireFormat::OpenAiChat, + &json!({ + "model": "gpt", + "messages": [{ + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": {"data": "AAAA", "format": "wav"} + }] + }] + }), + &TranslationPolicy::default(), + )?; + assert_eq!( + chat.request.input_modalities(), + [InputModality::Audio].into_iter().collect() + ); + + let responses = engine.decode_request( + WireFormat::OpenAiResponses, + &json!({ + "model": "gpt", + "input": [{ + "role": "user", + "content": [{ + "type": "input_video", + "video": {"media_type": "video/mp4", "data": "AAAA"} + }] + }] + }), + &TranslationPolicy::default(), + )?; + assert_eq!( + responses.request.input_modalities(), + [InputModality::Video].into_iter().collect() + ); + Ok(()) +} + +#[test] +fn responses_function_call_output_media_is_typed_for_modality_routing() -> TestResult { + let engine = TranslationEngine::default(); + let responses = engine.decode_request( + WireFormat::OpenAiResponses, + &json!({ + "model": "gpt", + "input": [ + { + "role": "user", + "content": "Inspect the current desk state." + }, + { + "type": "function_call", + "call_id": "call_view_image", + "name": "view_image", + "arguments": "{\"path\":\"/tmp/desk.png\"}" + }, + { + "type": "function_call_output", + "call_id": "call_view_image", + "output": [ + {"type": "input_text", "text": "Local image render"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=" + } + ] + } + ] + }), + &TranslationPolicy::default(), + )?; + + assert_eq!( + responses.request.input_modalities(), + [InputModality::Text, InputModality::Image] + .into_iter() + .collect() + ); + Ok(()) +} + // Verifies Anthropic-only request fields are dropped or mapped for OpenAI Chat. #[test] fn anthropic_request_translates_to_openai_chat_without_anthropic_only_fields() -> TestResult { @@ -1308,6 +1394,60 @@ fn openai_request_translates_system_developer_and_reasoning_to_anthropic() -> Te Ok(()) } +// Anthropic rejects data URLs in URL sources, so Responses images must become base64 blocks. +#[test] +fn responses_data_image_urls_become_anthropic_base64_sources() -> TestResult { + let body = json!({ + "model": "claude-fable-5", + "input": [{ + "role": "user", + "content": [ + {"type": "input_text", "text": "Compare these images."}, + { + "type": "input_image", + "image_url": "data:image/png;base64,aW1hZ2U=" + }, + { + "type": "input_image", + "image_url": "https://example.test/image.png" + } + ] + }] + }); + + let output = TranslationEngine::default() + .translate_request( + WireFormat::OpenAiResponses, + WireFormat::AnthropicMessages, + &body, + &TranslationPolicy::default(), + )? + .body; + + assert_eq!( + output["messages"][0]["content"][1], + json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aW1hZ2U=" + } + }) + ); + assert_eq!( + output["messages"][0]["content"][2], + json!({ + "type": "image", + "source": { + "type": "url", + "url": "https://example.test/image.png" + } + }) + ); + Ok(()) +} + // Verifies Anthropic receives its supported schema subset without mutating the neutral contract. #[test] fn openai_schema_constraints_are_removed_from_anthropic_output_format() -> TestResult { diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs index cc863ece3..4cd24e76d 100644 --- a/crates/switchyard-translation/tests/response_translation.rs +++ b/crates/switchyard-translation/tests/response_translation.rs @@ -496,6 +496,43 @@ fn openai_chat_usage_without_breakdowns_still_emits_responses_usage_details() -> Ok(()) } +// vLLM exposes hidden-thinking usage as a top-level extension rather than OpenAI's nested +// completion detail. Preserve it so classifier telemetry reports the actual reasoning work. +#[test] +fn openai_chat_top_level_reasoning_usage_translates_to_responses_details() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "id": "chatcmpl-vllm", + "model": "deepseek-v4-flash", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "{\"target\":\"luna\"}"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9093, + "completion_tokens": 347, + "total_tokens": 9440, + "reasoning_tokens": 339 + } + }); + + let output = engine + .translate_response( + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + assert_eq!( + output["usage"]["output_tokens_details"], + json!({"reasoning_tokens": 339}) + ); + Ok(()) +} + // Verifies a partial breakdown does not suppress the other detail object: an upstream that // reports cached tokens but no reasoning tokens must still carry both. #[test] diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 3aeed7159..9741de8ae 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -434,6 +434,61 @@ fn openai_chat_to_responses_uses_served_model_without_losing_source_model() -> T Ok(()) } +// Pi persists the ID from the final item, so it must match the ID announced at item start. +#[test] +fn translated_responses_message_id_survives_terminal_events() -> TestResult { + let engine = TranslationEngine::default(); + let mut state = + StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses); + let chunks = [ + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-upstream", + "choices": [{ + "index": 0, + "delta": {"content": "hello"}, + "finish_reason": null + }] + }), + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-upstream", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }), + ]; + + let mut events = Vec::new(); + for chunk in chunks { + events.extend(engine.translate_event( + &mut state, + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + &chunk, + )?); + } + events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?); + + let added_id = events + .iter() + .find(|event| event["type"] == "response.output_item.added") + .and_then(|event| event["item"]["id"].as_str()); + let done_id = events + .iter() + .find(|event| event["type"] == "response.output_item.done") + .and_then(|event| event["item"]["id"].as_str()); + let completed_id = events + .iter() + .find(|event| event["type"] == "response.completed") + .and_then(|event| event["response"]["output"][0]["id"].as_str()); + + assert_eq!(added_id, Some("msg_0")); + assert_eq!(done_id, added_id); + assert_eq!(completed_id, added_id); + Ok(()) +} + // Verifies OpenAI Chat finish emits a terminal chunk when the source closes without one. #[test] fn openai_chat_finish_synthesizes_terminal_chunk_after_incomplete_source() -> TestResult { @@ -549,6 +604,43 @@ fn openai_chat_stream_reasoning_usage_translates_to_responses_usage_details() -> Ok(()) } +#[test] +fn openai_chat_stream_top_level_reasoning_usage_translates_to_responses_details() -> TestResult { + let engine = TranslationEngine::default(); + let mut state = + StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses); + let usage = json!({ + "id": "chatcmpl-vllm", + "object": "chat.completion.chunk", + "model": "deepseek-v4-flash", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "reasoning_tokens": 3 + } + }); + + let mut events = engine.translate_event( + &mut state, + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + &usage, + )?; + events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?); + + let completed = events + .iter() + .find(|event| event["type"] == "response.completed") + .ok_or("expected final Responses completion event")?; + assert_eq!( + completed["response"]["usage"]["output_tokens_details"], + json!({"reasoning_tokens": 3}) + ); + Ok(()) +} + // Verifies streamed cache usage reaches Responses clients in the standard details object. #[test] fn openai_chat_stream_cache_usage_translates_to_responses_usage_details() -> TestResult { diff --git a/docs/cli_reference.md b/docs/cli_reference.md index a8a0e583c..a47355b08 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -68,6 +68,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | +| `--routing-log-classifier-content` | `false` | Include normalized classifier prompts, model reasoning, and verdicts in the routing log. Requires `--routing-log-file`. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 68d01bc54..0680fa633 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -47,6 +47,9 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. | | `forward_auth` | No | `false` | Forward the caller's provider credential to this upstream. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | +| `bridge_custom_tools` | No | `false` | For an `openai_responses` backend without native custom-tool support, bridge custom definitions and replay items through function tools, then restore custom-tool responses. | +| `eager_load_tool_search` | No | `false` | For an `openai_responses` backend without native `tool_search`, remove discovery records and expose deferred tools eagerly. | +| `xai_responses_compatibility` | No | `false` | Normalize cross-provider Responses replay and `web_search` options to xAI's supported schema. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | The TOML never contains the secret itself. `api_key_env` names a variable that @@ -76,14 +79,68 @@ server rejects an Anthropic forwarding route called through an OpenAI endpoint, or an OpenAI forwarding route called through an Anthropic endpoint, before it calls an upstream. +Set `bridge_custom_tools = true` only for a Responses-compatible provider that +accepts function tools but rejects `type = "custom"`. Native custom-tool +providers should leave it disabled so grammar-constrained tools pass through +unchanged. Other client formats reject this setting. + +Set `eager_load_tool_search = true` only for a Responses-compatible provider +that rejects `type = "tool_search"`. Switchyard removes the discovery-only tool +and replay items, strips `defer_loading`, and flattens deferred namespaces so +the same client tools remain callable immediately. Providers with native tool +search should leave it disabled. Other client formats reject this setting. + +Set `xai_responses_compatibility = true` for an xAI Responses backend. It keeps +live `web_search` available while removing OpenAI-only options such as +`external_web_access`; a definition with `external_web_access = false` is removed +rather than silently upgraded to live search. Other client formats reject this +setting. + +For every Responses backend, Switchyard removes provider-bound +`encrypted_content` from replayed reasoning items while retaining their portable +summaries. This permits a dynamic route to move a conversation between Responses +providers without asking one provider to verify another provider's ciphertext. + +When a Responses request containing `web_search` is translated to an Anthropic +backend, Switchyard emits Anthropic's native `web_search_20250305` server tool. +Other Responses backends retain their original tool shape unless the xAI +compatibility setting is enabled. + ## `[targets.]` | Key | Required | Default | Meaning | |---|:---:|---|---| | `id` | Yes | — | Exact model ID sent upstream. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | +| `input_modalities` | No | unset | Non-empty, duplicate-free list of accepted inputs: `text`, `image`, `audio`, `video`, and/or `file`. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | +Modality-aware routing is opt-in per route. Either every completion target in a +route declares `input_modalities`, or none may declare it. Judge-only targets do +not participate in this completeness rule or in the route's advertised +capabilities; when a judge does declare modalities, its list must include +`text` because Switchyard sends it a text verdict prompt. + +```toml +[targets.text] +id = "deepseek/model" +llm_client = "local" +input_modalities = ["text"] + +[targets.vision] +id = "qwen/vision" +llm_client = "local" +input_modalities = ["text", "image"] +``` + +Switchyard considers typed content in instructions, conversation history, and +nested tool results. A target is eligible only when it supports every modality +present in the request; unsupported content is not removed. If no target is +eligible, the request returns HTTP 400 without an upstream call. `GET /v1/models` +advertises the canonical union of the completion targets' declarations. Routes +without declarations, and `noop` routes, advertise `text` and retain their +existing routing behavior. + ## `[routes.]` Every route takes the common keys below, plus the keys for its type. @@ -95,6 +152,7 @@ Every route takes the common keys below, plus the keys for its type. | `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. | | `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. | | `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. | +| `web_search` | No | unset | Whether `GET /v1/models` advertises provider-hosted or translated web-search support. Unset routes are advertised to Codex with search disabled. | ### `noop` @@ -142,6 +200,7 @@ Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. |---|:---:|---|---| | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Yes | — | Target the judge is called through. Not a routing destination. | +| `target_failover` | No | `true` | When false, a failed selected completion target returns its error instead of trying another route target. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | | `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | @@ -155,6 +214,7 @@ Capability mode classifies before serving. See | `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. | | `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `session_affinity` | No | `false` | Reuses a session's first decision on later turns. | +| `turn_affinity` | No | `false` | Reuses a decision for tool-loop continuations until the next human user message. Requires a session identity and cannot be combined with `session_affinity`. Modality filtering may replace an incompatible assignment. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `session_affinity`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | @@ -184,6 +244,7 @@ policy selector, and routes to any configured target label. | `response_schema` | Yes | — | Inner JSON Schema encoded as a TOML string. Switchyard adds the provider wrapper. | | `policy` | Yes | — | Policy table. `target_selector` accepts a JSON Pointer such as `/decision/target`. | | `session_affinity` | No | `false` | Reuses a session's first decision on later turns. | +| `turn_affinity` | No | `false` | Reuses a decision for tool-loop continuations until the next human user message. Requires a session identity and cannot be combined with `session_affinity`. Modality filtering may replace an incompatible assignment. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `session_affinity`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index a1cc43e45..dda2ff325 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -105,8 +105,9 @@ for the server merge behavior. |---|---|---| | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | -| `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `recent_turn_window` | unset | When unset, the judge sees the opening user task and latest human user follow-up when they differ; tool results and provider-native tool items are excluded. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | | `session_affinity` | `false` | Retains the first selected target for a session and reuses it on later requests. | +| `turn_affinity` | `false` | Retains a target across tool-loop continuations, then classifies again for the next human user message. An incompatible retained target is replaced by a modality-compatible decision. Cannot be combined with `session_affinity`. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `session_affinity = true`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | | `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. | @@ -213,6 +214,15 @@ Affinity is process-local. Clients can send `x-switchyard-session-id`, or enable `message_hash_fallback` to key requests without session metadata from the first user-message text. +With `turn_affinity = true`, only requests belonging to the same human user turn +reuse the selected target. An explicit `x-switchyard-turn-id` is used when +available; otherwise Switchyard fingerprints the human user messages within the +session while excluding tool results and native tool-loop items. The next user +message is classified again. Modality eligibility remains authoritative, so a +new image or other supported modality can replace an incompatible retained +target within the turn. Turn affinity requires a session identity and cannot be +combined with `session_affinity`. + ## Run the route After [installing the Rust server](../getting_started.md#install-the-server), export diff --git a/switchyard/cli/launchers/codex_cli_launcher.py b/switchyard/cli/launchers/codex_cli_launcher.py index d3b1aced7..ef8f8af02 100644 --- a/switchyard/cli/launchers/codex_cli_launcher.py +++ b/switchyard/cli/launchers/codex_cli_launcher.py @@ -167,6 +167,7 @@ def _run_codex_with_switchyard( try: try: caller_auth = server.caller_auth_kind(display_model) + input_modalities = server.input_modalities(display_model) except ValueError as error: logger.error("%s", error) return 1 @@ -176,7 +177,19 @@ def _run_codex_with_switchyard( ) return 1 use_openai_auth = caller_auth == "openai" - model_catalog_json = _write_codex_model_catalog(codex_bin, codex_model_catalog) + modalities_by_model = {display_model: input_modalities} + for entry_model, _display, _description in codex_model_catalog: + if entry_model in modalities_by_model: + continue + try: + modalities_by_model[entry_model] = server.input_modalities(entry_model) + except ValueError: + continue + model_catalog_json = _write_codex_model_catalog( + codex_bin, + codex_model_catalog, + input_modalities_by_model=modalities_by_model, + ) command = _codex_command( codex_bin, codex_args, diff --git a/switchyard/cli/launchers/codex_model_catalog.py b/switchyard/cli/launchers/codex_model_catalog.py index 12e5acd71..3bb2ab5c6 100644 --- a/switchyard/cli/launchers/codex_model_catalog.py +++ b/switchyard/cli/launchers/codex_model_catalog.py @@ -17,7 +17,7 @@ import os import subprocess import tempfile -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Any, TypeAlias logger = logging.getLogger(__name__) @@ -106,6 +106,7 @@ def _load_codex_model_template(codex_bin: str) -> dict[str, Any]: def _build_codex_model_catalog( codex_bin: str, entries: Sequence[CodexModelCatalogEntry], + input_modalities_by_model: Mapping[str, Sequence[str]] | None = None, ) -> dict[str, list[dict[str, Any]]]: """Build Codex catalog JSON for Switchyard route ids.""" template = _load_codex_model_template(codex_bin) @@ -120,6 +121,8 @@ def _build_codex_model_catalog( model["supported_in_api"] = True model["availability_nux"] = None model["upgrade"] = None + modalities = (input_modalities_by_model or {}).get(model_id, ("text",)) + model["input_modalities"] = list(modalities) models.append(model) return {"models": models} @@ -127,12 +130,17 @@ def _build_codex_model_catalog( def _write_codex_model_catalog( codex_bin: str, entries: Sequence[CodexModelCatalogEntry], + input_modalities_by_model: Mapping[str, Sequence[str]] | None = None, ) -> str | None: """Write a temporary Codex catalog file and return its path.""" if not entries: return None - catalog = _build_codex_model_catalog(codex_bin, entries) + catalog = _build_codex_model_catalog( + codex_bin, + entries, + input_modalities_by_model=input_modalities_by_model, + ) with tempfile.NamedTemporaryFile( "w", encoding="utf-8", diff --git a/switchyard/cli/launchers/native_server.py b/switchyard/cli/launchers/native_server.py index d33e987c5..3dd5d81b3 100644 --- a/switchyard/cli/launchers/native_server.py +++ b/switchyard/cli/launchers/native_server.py @@ -50,6 +50,10 @@ def caller_auth_kind(self, model: str) -> str | None: """Return which caller credential the route forwards, if any.""" return self._server.caller_auth_kind(model) + def input_modalities(self, model: str) -> list[str]: + """Return the route's discovered input modalities.""" + return self._server.input_modalities(model) + def close(self) -> None: """Gracefully stop the native server.""" self._server.close() diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 67e4e1888..4bc19b543 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -94,6 +94,7 @@ def __init__( *, threshold_step: float = 0.0, session_affinity: bool = False, + turn_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, max_output_tokens: int = 4096, diff --git a/switchyard_rust/server.py b/switchyard_rust/server.py index a30577eca..bb8ef3efc 100644 --- a/switchyard_rust/server.py +++ b/switchyard_rust/server.py @@ -26,6 +26,8 @@ def base_url(self) -> str: ... def caller_auth_kind(self, model: str) -> str | None: ... + def input_modalities(self, model: str) -> list[str]: ... + def close(self, timeout_secs: float = 2.0) -> None: ... def __enter__(self) -> Self: ... diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 6873c7b75..c5768dba3 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -11,6 +11,7 @@ from switchyard.cli.launch_command import _config_path from switchyard.cli.launchers.claude_code_launcher import _claude_env from switchyard.cli.launchers.codex_cli_launcher import _codex_env, _provider_overrides +from switchyard.cli.launchers.codex_model_catalog import _build_codex_model_catalog from switchyard.cli.launchers.native_server import NativeServer from switchyard.cli.switchyard_cli import _build_parser @@ -106,23 +107,168 @@ def caller_auth_kind(self, model: str) -> str | None: captured["model"] = model return "anthropic" + def input_modalities(self, model: str) -> list[str]: + captured["modalities_model"] = model + return ["text", "image"] + import switchyard_rust.server monkeypatch.setattr(switchyard_rust.server, "Server", FakeServer) server = NativeServer(config) assert server.caller_auth_kind("switchyard/route") == "anthropic" + assert server.input_modalities("switchyard/route") == ["text", "image"] server.close() assert captured == { "path": config, "port": 0, "model": "switchyard/route", + "modalities_model": "switchyard/route", "closed": True, } assert server.port == 4321 assert config.exists() +def test_codex_catalog_uses_route_modalities_and_defaults_undeclared_to_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import switchyard.cli.launchers.codex_model_catalog as catalog_module + + monkeypatch.setattr( + catalog_module, + "_load_codex_model_template", + lambda _codex_bin: {"input_modalities": ["text", "image", "audio"]}, + ) + entries = [("switchyard/route", "Route", "Test route")] + + discovered = _build_codex_model_catalog( + "codex", + entries, + input_modalities_by_model={"switchyard/route": ["text", "image"]}, + ) + undeclared = _build_codex_model_catalog("codex", entries) + + assert discovered["models"][0]["input_modalities"] == ["text", "image"] + assert undeclared["models"][0]["input_modalities"] == ["text"] + + +def test_codex_launcher_discovers_modalities_for_every_catalog_entry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import switchyard.cli.launchers.codex_cli_launcher as launcher + + class FakeServer: + port = 4321 + stats = object() + + def __init__(self) -> None: + self.queried_models: list[str] = [] + self.closed = False + + def caller_auth_kind(self, model: str) -> str | None: + assert model == "switchyard/text" + return None + + def input_modalities(self, model: str) -> list[str]: + self.queried_models.append(model) + return { + "switchyard/text": ["text"], + "switchyard/vision": ["text", "image"], + }[model] + + def close(self) -> None: + self.closed = True + + server = FakeServer() + captured_modalities: dict[str, list[str]] = {} + + def write_catalog( + _codex_bin: str, + _entries: object, + input_modalities_by_model: dict[str, list[str]] | None = None, + ) -> None: + captured_modalities.update(input_modalities_by_model or {}) + + monkeypatch.setattr(launcher, "_find_codex_binary", lambda: "codex") + monkeypatch.setattr(launcher, "silence_launch_loggers", lambda **_kwargs: None) + monkeypatch.setattr( + launcher, + "configure_debug_file_logging", + lambda **_kwargs: tmp_path / "switchyard.log", + ) + monkeypatch.setattr(launcher, "_start_native_server", lambda _config: server) + monkeypatch.setattr(launcher, "_write_codex_model_catalog", write_catalog) + monkeypatch.setattr(launcher, "_wait_ready", lambda _port: True) + monkeypatch.setattr(launcher, "print_ready_banner", lambda **_kwargs: None) + monkeypatch.setattr(launcher, "stdin_is_tty", lambda: False) + monkeypatch.setattr(launcher, "_supervise_codex", lambda _command, _env: 0) + monkeypatch.setattr(launcher, "print_session_summary", lambda _stats: None) + monkeypatch.setattr(launcher, "_remove_codex_model_catalog", lambda _path: None) + + result = launcher._run_codex_with_switchyard( + tmp_path / "routes.toml", + display_model="switchyard/text", + codex_args=[], + codex_model_catalog=[ + ("switchyard/text", "Text", "Text route"), + ("switchyard/vision", "Vision", "Vision route"), + ], + ) + + assert result == 0 + assert server.queried_models == ["switchyard/text", "switchyard/vision"] + assert captured_modalities == { + "switchyard/text": ["text"], + "switchyard/vision": ["text", "image"], + } + assert server.closed + + +def test_native_server_exposes_route_derived_modalities(tmp_path: Path) -> None: + config = tmp_path / "modalities.toml" + config.write_text( + """ +schema_version = 1 + +[llm_clients.local] +format = "openai_chat" +base_url = "http://127.0.0.1:9/v1" + +[targets.text] +id = "model/text" +llm_client = "local" +input_modalities = ["text"] + +[targets.vision] +id = "model/vision" +llm_client = "local" +input_modalities = ["image", "text"] + +[targets.legacy] +id = "model/legacy" +llm_client = "local" + +[routes.multimodal] +id = "switchyard/multimodal" +type = "random" +targets = ["text", "vision"] + +[routes.legacy] +id = "switchyard/legacy" +type = "passthrough" +target = "legacy" +""" + ) + server = NativeServer(config) + try: + assert server.input_modalities("switchyard/multimodal") == ["text", "image"] + assert server.input_modalities("switchyard/legacy") == ["text"] + finally: + server.close() + + def test_missing_explicit_config_is_a_cli_error(tmp_path: Path) -> None: missing = tmp_path / "missing.toml" with pytest.raises(SystemExit, match="config file not found"):