diff --git a/src/providers/compatible.rs b/src/providers/compatible.rs index 8ff54be4b..3a4bed581 100644 --- a/src/providers/compatible.rs +++ b/src/providers/compatible.rs @@ -388,6 +388,37 @@ impl OpenAiCompatibleProvider { }) .collect() } + + fn openai_tools_to_tool_specs(tools: &[serde_json::Value]) -> Vec { + tools + .iter() + .filter_map(|tool| { + let function = tool.get("function")?; + let name = function.get("name")?.as_str()?.trim(); + if name.is_empty() { + return None; + } + + let description = function + .get("description") + .and_then(|value| value.as_str()) + .unwrap_or("No description provided") + .to_string(); + let parameters = function.get("parameters").cloned().unwrap_or_else(|| { + serde_json::json!({ + "type": "object", + "properties": {} + }) + }); + + Some(crate::tools::ToolSpec { + name: name.to_string(), + description, + parameters, + }) + }) + .collect() + } } #[derive(Debug, Serialize)] @@ -1584,24 +1615,27 @@ impl OpenAiCompatibleProvider { } fn is_native_tool_schema_unsupported(status: reqwest::StatusCode, error: &str) -> bool { - if !matches!( - status, - reqwest::StatusCode::BAD_REQUEST | reqwest::StatusCode::UNPROCESSABLE_ENTITY - ) { - return false; - } + super::is_native_tool_schema_rejection(status, error) + } - let lower = error.to_lowercase(); - [ - "unknown parameter: tools", - "unsupported parameter: tools", - "unrecognized field `tools`", - "does not support tools", - "function calling is not supported", - "tool_choice", - ] - .iter() - .any(|hint| lower.contains(hint)) + async fn prompt_guided_tools_fallback( + &self, + messages: &[ChatMessage], + tools: Option<&[crate::tools::ToolSpec]>, + model: &str, + temperature: f64, + ) -> anyhow::Result { + let fallback_messages = Self::with_prompt_guided_tool_instructions(messages, tools); + let text = self + .chat_with_history(&fallback_messages, model, temperature) + .await?; + Ok(ProviderChatResponse { + text: Some(text), + tool_calls: vec![], + usage: None, + reasoning_content: None, + quota_metadata: None, + }) } } @@ -1955,6 +1989,21 @@ impl Provider for OpenAiCompatibleProvider { if !response.status().is_success() { let status = response.status(); + let error = response.text().await?; + let sanitized = super::sanitize_api_error(&error); + + if Self::is_native_tool_schema_unsupported(status, &error) { + let fallback_tool_specs = Self::openai_tools_to_tool_specs(tools); + return self + .prompt_guided_tools_fallback( + messages, + (!fallback_tool_specs.is_empty()).then_some(fallback_tool_specs.as_slice()), + model, + temperature, + ) + .await; + } + if status == reqwest::StatusCode::NOT_FOUND && self.supports_responses_fallback { return self .chat_via_responses_chat( @@ -1965,7 +2014,8 @@ impl Provider for OpenAiCompatibleProvider { ) .await; } - return Err(super::api_error(&self.name, response).await); + + anyhow::bail!("{} API error ({status}): {sanitized}", self.name); } let body = response.text().await?; @@ -2090,19 +2140,15 @@ impl Provider for OpenAiCompatibleProvider { let error = response.text().await?; let sanitized = super::sanitize_api_error(&error); - if Self::is_native_tool_schema_unsupported(status, &sanitized) { - let fallback_messages = - Self::with_prompt_guided_tool_instructions(request.messages, request.tools); - let text = self - .chat_with_history(&fallback_messages, model, temperature) - .await?; - return Ok(ProviderChatResponse { - text: Some(text), - tool_calls: vec![], - usage: None, - reasoning_content: None, - quota_metadata: None, - }); + if Self::is_native_tool_schema_unsupported(status, &error) { + return self + .prompt_guided_tools_fallback( + request.messages, + request.tools, + model, + temperature, + ) + .await; } if status == reqwest::StatusCode::NOT_FOUND && self.supports_responses_fallback { @@ -2273,6 +2319,10 @@ impl Provider for OpenAiCompatibleProvider { #[cfg(test)] mod tests { use super::*; + use axum::{extract::State, http::StatusCode, routing::post, Json, Router}; + use serde_json::Value; + use std::sync::Arc; + use tokio::sync::Mutex; fn make_provider(name: &str, url: &str, key: Option<&str>) -> OpenAiCompatibleProvider { OpenAiCompatibleProvider::new(name, url, key, AuthStyle::Bearer) @@ -2972,12 +3022,32 @@ mod tests { reqwest::StatusCode::BAD_REQUEST, "unknown parameter: tools" )); + assert!(OpenAiCompatibleProvider::is_native_tool_schema_unsupported( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "unknown parameter: tools" + )); assert!( !OpenAiCompatibleProvider::is_native_tool_schema_unsupported( reqwest::StatusCode::UNAUTHORIZED, "unknown parameter: tools" ) ); + assert!( + !OpenAiCompatibleProvider::is_native_tool_schema_unsupported( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "upstream gateway unavailable" + ) + ); + assert!( + !OpenAiCompatibleProvider::is_native_tool_schema_unsupported( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "tool_choice was set to auto by default policy" + ) + ); + assert!(OpenAiCompatibleProvider::is_native_tool_schema_unsupported( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "mapper validation failed: tool schema is incompatible" + )); } #[test] @@ -3155,6 +3225,30 @@ mod tests { assert_eq!(tools[0]["function"]["parameters"]["required"][0], "command"); } + #[test] + fn openai_tools_convert_back_to_tool_specs_for_prompt_fallback() { + let openai_tools = vec![serde_json::json!({ + "type": "function", + "function": { + "name": "weather_lookup", + "description": "Look up weather by city", + "parameters": { + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"] + } + } + })]; + + let specs = OpenAiCompatibleProvider::openai_tools_to_tool_specs(&openai_tools); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].name, "weather_lookup"); + assert_eq!(specs[0].description, "Look up weather by city"); + assert_eq!(specs[0].parameters["required"][0], "city"); + } + #[test] fn request_serializes_with_tools() { let tools = vec![serde_json::json!({ @@ -3291,6 +3385,393 @@ mod tests { .contains("TestProvider API key not set")); } + #[tokio::test] + async fn chat_with_tools_falls_back_on_http_516_tool_schema_error() { + #[derive(Clone, Default)] + struct NativeToolFallbackState { + requests: Arc>>, + } + + async fn chat_endpoint( + State(state): State, + Json(payload): Json, + ) -> (StatusCode, Json) { + state.requests.lock().await.push(payload.clone()); + + if payload.get("tools").is_some() { + let long_mapper_prefix = "x".repeat(260); + let error_message = format!("{long_mapper_prefix} unknown parameter: tools"); + return ( + StatusCode::from_u16(516).expect("516 is a valid HTTP status"), + Json(serde_json::json!({ + "error": { + "message": error_message + } + })), + ); + } + + ( + StatusCode::OK, + Json(serde_json::json!({ + "choices": [{ + "message": { + "content": "CALL weather_lookup {\"city\":\"Paris\"}" + } + }] + })), + ) + } + + let state = NativeToolFallbackState::default(); + let app = Router::new() + .route("/chat/completions", post(chat_endpoint)) + .with_state(state.clone()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("server local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve test app"); + }); + + let provider = make_provider( + "TestProvider", + &format!("http://{}", addr), + Some("test-provider-key"), + ); + let messages = vec![ChatMessage::user("check weather")]; + let tools = vec![serde_json::json!({ + "type": "function", + "function": { + "name": "weather_lookup", + "description": "Look up weather by city", + "parameters": { + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"] + } + } + })]; + + let result = provider + .chat_with_tools(&messages, &tools, "test-model", 0.7) + .await + .expect("516 tool-schema rejection should trigger prompt-guided fallback"); + + assert_eq!( + result.text.as_deref(), + Some("CALL weather_lookup {\"city\":\"Paris\"}") + ); + assert!( + result.tool_calls.is_empty(), + "prompt-guided fallback should return text without native tool_calls" + ); + + let requests = state.requests.lock().await; + assert_eq!( + requests.len(), + 2, + "expected native attempt + fallback attempt" + ); + + assert!( + requests[0].get("tools").is_some(), + "native attempt must include tools schema" + ); + assert_eq!( + requests[0].get("tool_choice").and_then(|v| v.as_str()), + Some("auto") + ); + + assert!( + requests[1].get("tools").is_none(), + "fallback request should not include native tools" + ); + assert!( + requests[1].get("tool_choice").is_none(), + "fallback request should omit native tool_choice" + ); + let fallback_messages = requests[1] + .get("messages") + .and_then(|v| v.as_array()) + .expect("fallback request should include messages"); + let fallback_system = fallback_messages + .iter() + .find(|m| m.get("role").and_then(|r| r.as_str()) == Some("system")) + .expect("fallback should prepend system tool instructions"); + let fallback_system_text = fallback_system + .get("content") + .and_then(|v| v.as_str()) + .expect("fallback system prompt should be plain text"); + assert!(fallback_system_text.contains("Available Tools")); + assert!(fallback_system_text.contains("weather_lookup")); + + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn chat_falls_back_on_http_516_tool_schema_error() { + #[derive(Clone, Default)] + struct NativeToolFallbackState { + requests: Arc>>, + } + + async fn chat_endpoint( + State(state): State, + Json(payload): Json, + ) -> (StatusCode, Json) { + state.requests.lock().await.push(payload.clone()); + + if payload.get("tools").is_some() { + let long_mapper_prefix = "x".repeat(260); + let error_message = + format!("{long_mapper_prefix} mapper validation failed: tool schema mismatch"); + return ( + StatusCode::from_u16(516).expect("516 is a valid HTTP status"), + Json(serde_json::json!({ + "error": { + "message": error_message + } + })), + ); + } + + ( + StatusCode::OK, + Json(serde_json::json!({ + "choices": [{ + "message": { + "content": "CALL weather_lookup {\"city\":\"Paris\"}" + } + }] + })), + ) + } + + let state = NativeToolFallbackState::default(); + let app = Router::new() + .route("/chat/completions", post(chat_endpoint)) + .with_state(state.clone()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("server local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve test app"); + }); + + let provider = make_provider( + "TestProvider", + &format!("http://{}", addr), + Some("test-provider-key"), + ); + let messages = vec![ChatMessage::user("check weather")]; + let tools = vec![crate::tools::ToolSpec { + name: "weather_lookup".to_string(), + description: "Look up weather by city".to_string(), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"] + }), + }]; + + let result = provider + .chat( + ProviderChatRequest { + messages: &messages, + tools: Some(&tools), + }, + "test-model", + 0.7, + ) + .await + .expect("chat() should fallback on HTTP 516 mapper tool-schema rejection"); + + assert_eq!( + result.text.as_deref(), + Some("CALL weather_lookup {\"city\":\"Paris\"}") + ); + assert!( + result.tool_calls.is_empty(), + "prompt-guided fallback should return text without native tool_calls" + ); + + let requests = state.requests.lock().await; + assert_eq!( + requests.len(), + 2, + "expected native attempt + fallback attempt" + ); + assert!( + requests[0].get("tools").is_some(), + "native attempt must include tools schema" + ); + assert!( + requests[1].get("tools").is_none(), + "fallback request should not include native tools" + ); + let fallback_messages = requests[1] + .get("messages") + .and_then(|v| v.as_array()) + .expect("fallback request should include messages"); + let fallback_system = fallback_messages + .iter() + .find(|m| m.get("role").and_then(|r| r.as_str()) == Some("system")) + .expect("fallback should prepend system tool instructions"); + let fallback_system_text = fallback_system + .get("content") + .and_then(|v| v.as_str()) + .expect("fallback system prompt should be plain text"); + assert!(fallback_system_text.contains("Available Tools")); + assert!(fallback_system_text.contains("weather_lookup")); + + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn chat_with_tools_does_not_fallback_on_generic_516() { + #[derive(Clone, Default)] + struct Generic516State { + requests: Arc>>, + } + + async fn chat_endpoint( + State(state): State, + Json(payload): Json, + ) -> (StatusCode, Json) { + state.requests.lock().await.push(payload); + ( + StatusCode::from_u16(516).expect("516 is a valid HTTP status"), + Json(serde_json::json!({ + "error": { "message": "upstream gateway unavailable" } + })), + ) + } + + let state = Generic516State::default(); + let app = Router::new() + .route("/chat/completions", post(chat_endpoint)) + .with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("server local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve test app"); + }); + + let provider = make_provider( + "TestProvider", + &format!("http://{}", addr), + Some("test-provider-key"), + ); + let messages = vec![ChatMessage::user("check weather")]; + let tools = vec![serde_json::json!({ + "type": "function", + "function": { + "name": "weather_lookup", + "description": "Look up weather by city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + } + } + })]; + + let err = provider + .chat_with_tools(&messages, &tools, "test-model", 0.7) + .await + .expect_err("generic 516 must not trigger prompt-guided fallback"); + assert!(err.to_string().contains("API error (516")); + + let requests = state.requests.lock().await; + assert_eq!(requests.len(), 1, "must not issue fallback retry request"); + assert!(requests[0].get("tools").is_some()); + + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn chat_does_not_fallback_on_generic_516() { + #[derive(Clone, Default)] + struct Generic516State { + requests: Arc>>, + } + + async fn chat_endpoint( + State(state): State, + Json(payload): Json, + ) -> (StatusCode, Json) { + state.requests.lock().await.push(payload); + ( + StatusCode::from_u16(516).expect("516 is a valid HTTP status"), + Json(serde_json::json!({ + "error": { "message": "upstream gateway unavailable" } + })), + ) + } + + let state = Generic516State::default(); + let app = Router::new() + .route("/chat/completions", post(chat_endpoint)) + .with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("server local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve test app"); + }); + + let provider = make_provider( + "TestProvider", + &format!("http://{}", addr), + Some("test-provider-key"), + ); + let messages = vec![ChatMessage::user("check weather")]; + let tools = vec![crate::tools::ToolSpec { + name: "weather_lookup".to_string(), + description: "Look up weather by city".to_string(), + parameters: serde_json::json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + }), + }]; + + let err = provider + .chat( + ProviderChatRequest { + messages: &messages, + tools: Some(&tools), + }, + "test-model", + 0.7, + ) + .await + .expect_err("generic 516 must not trigger prompt-guided fallback"); + assert!(err.to_string().contains("API error (516")); + + let requests = state.requests.lock().await; + assert_eq!(requests.len(), 1, "must not issue fallback retry request"); + assert!(requests[0].get("tools").is_some()); + + server.abort(); + let _ = server.await; + } + #[test] fn response_with_no_tool_calls_has_empty_vec() { let json = r#"{"choices":[{"message":{"content":"Just text, no tools."}}]}"#; diff --git a/src/providers/mod.rs b/src/providers/mod.rs index adf6124dd..d4a0cf431 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -819,6 +819,57 @@ pub fn sanitize_api_error(input: &str) -> String { format!("{}...", &scrubbed[..end]) } +/// True when HTTP status indicates request-shape/schema rejection for native tools. +/// +/// 516 is included for OpenAI-compatible providers that surface mapper/schema +/// errors via vendor-specific status codes instead of standard 4xx. +pub(crate) fn is_native_tool_schema_rejection_status(status: reqwest::StatusCode) -> bool { + matches!( + status, + reqwest::StatusCode::BAD_REQUEST | reqwest::StatusCode::UNPROCESSABLE_ENTITY + ) || status.as_u16() == 516 +} + +/// Detect request-mapper/tool-schema incompatibility hints in provider errors. +pub(crate) fn has_native_tool_schema_rejection_hint(error: &str) -> bool { + let lower = error.to_lowercase(); + + let direct_hints = [ + "unknown parameter: tools", + "unsupported parameter: tools", + "unrecognized field `tools`", + "does not support tools", + "function calling is not supported", + "unknown parameter: tool_choice", + "unsupported parameter: tool_choice", + "unrecognized field `tool_choice`", + "invalid parameter: tool_choice", + ]; + if direct_hints.iter().any(|hint| lower.contains(hint)) { + return true; + } + + let mapper_tool_schema_hint = lower.contains("mapper") + && (lower.contains("tool") || lower.contains("function")) + && (lower.contains("schema") + || lower.contains("parameter") + || lower.contains("validation")); + if mapper_tool_schema_hint { + return true; + } + + lower.contains("tool schema") + && (lower.contains("mismatch") + || lower.contains("unsupported") + || lower.contains("invalid") + || lower.contains("incompatible")) +} + +/// Combined predicate for native tool-schema rejection. +pub(crate) fn is_native_tool_schema_rejection(status: reqwest::StatusCode, error: &str) -> bool { + is_native_tool_schema_rejection_status(status) && has_native_tool_schema_rejection_hint(error) +} + /// Build a sanitized provider error from a failed HTTP response. pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::Error { let status = response.status(); @@ -3037,6 +3088,67 @@ mod tests { // ── API error sanitization ─────────────────────────────── + #[test] + fn native_tool_schema_rejection_status_covers_vendor_516() { + assert!(is_native_tool_schema_rejection_status( + reqwest::StatusCode::BAD_REQUEST + )); + assert!(is_native_tool_schema_rejection_status( + reqwest::StatusCode::UNPROCESSABLE_ENTITY + )); + assert!(is_native_tool_schema_rejection_status( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code") + )); + assert!(!is_native_tool_schema_rejection_status( + reqwest::StatusCode::INTERNAL_SERVER_ERROR + )); + } + + #[test] + fn native_tool_schema_rejection_hint_is_precise() { + assert!(has_native_tool_schema_rejection_hint( + "unknown parameter: tools" + )); + assert!(has_native_tool_schema_rejection_hint( + "mapper validation failed: tool schema is incompatible" + )); + let long_prefix = "x".repeat(300); + let long_hint = format!("{long_prefix} unknown parameter: tools"); + assert!(has_native_tool_schema_rejection_hint(&long_hint)); + assert!(!has_native_tool_schema_rejection_hint( + "upstream gateway unavailable" + )); + assert!(!has_native_tool_schema_rejection_hint( + "temporary network timeout while contacting provider" + )); + assert!(!has_native_tool_schema_rejection_hint( + "tool_choice was set to auto by default policy" + )); + assert!(!has_native_tool_schema_rejection_hint( + "available tools: shell, weather, browser" + )); + } + + #[test] + fn native_tool_schema_rejection_combines_status_and_hint() { + assert!(is_native_tool_schema_rejection( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "unknown parameter: tools" + )); + assert!(is_native_tool_schema_rejection( + reqwest::StatusCode::BAD_REQUEST, + "unsupported parameter: tool_choice" + )); + assert!(!is_native_tool_schema_rejection( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + "unknown parameter: tools" + )); + assert!(!is_native_tool_schema_rejection( + reqwest::StatusCode::from_u16(516).expect("516 is a valid status code"), + "upstream gateway unavailable" + )); + } + #[test] fn sanitize_scrubs_sk_prefix() { let input = "request failed: sk-1234567890abcdef"; diff --git a/src/providers/reliable.rs b/src/providers/reliable.rs index b5e47e7c4..56eee0bde 100644 --- a/src/providers/reliable.rs +++ b/src/providers/reliable.rs @@ -20,6 +20,15 @@ fn is_non_retryable(err: &anyhow::Error) -> bool { return true; } + let msg = err.to_string(); + let msg_lower = msg.to_lowercase(); + + // Tool-schema/mapper incompatibility (including vendor 516 wrappers) + // is deterministic: retries won't fix an unsupported request shape. + if super::has_native_tool_schema_rejection_hint(&msg_lower) { + return true; + } + // 4xx errors are generally non-retryable (bad request, auth failure, etc.), // except 429 (rate-limit — transient) and 408 (timeout — worth retrying). if let Some(reqwest_err) = err.downcast_ref::() { @@ -30,7 +39,6 @@ fn is_non_retryable(err: &anyhow::Error) -> bool { } // Fallback: parse status codes from stringified errors (some providers // embed codes in error messages rather than returning typed HTTP errors). - let msg = err.to_string(); for word in msg.split(|c: char| !c.is_ascii_digit()) { if let Ok(code) = word.parse::() { if (400..500).contains(&code) { @@ -41,7 +49,6 @@ fn is_non_retryable(err: &anyhow::Error) -> bool { // Heuristic: detect auth/model failures by keyword when no HTTP status // is available (e.g. gRPC or custom transport errors). - let msg_lower = msg.to_lowercase(); let auth_failure_hints = [ "invalid api key", "incorrect api key", @@ -1137,6 +1144,9 @@ mod tests { assert!(is_non_retryable(&anyhow::anyhow!("401 Unauthorized"))); assert!(is_non_retryable(&anyhow::anyhow!("403 Forbidden"))); assert!(is_non_retryable(&anyhow::anyhow!("404 Not Found"))); + assert!(is_non_retryable(&anyhow::anyhow!( + "516 mapper tool schema mismatch: unknown parameter: tools" + ))); assert!(is_non_retryable(&anyhow::anyhow!( "invalid api key provided" ))); @@ -1153,6 +1163,9 @@ mod tests { "500 Internal Server Error" ))); assert!(!is_non_retryable(&anyhow::anyhow!("502 Bad Gateway"))); + assert!(!is_non_retryable(&anyhow::anyhow!( + "516 upstream gateway temporarily unavailable" + ))); assert!(!is_non_retryable(&anyhow::anyhow!("timeout"))); assert!(!is_non_retryable(&anyhow::anyhow!("connection reset"))); assert!(!is_non_retryable(&anyhow::anyhow!( @@ -1750,6 +1763,61 @@ mod tests { ); } + #[tokio::test] + async fn native_tool_schema_rejection_skips_retries_for_516() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider = ReliableProvider::new( + vec![( + "primary".into(), + Box::new(MockProvider { + calls: Arc::clone(&calls), + fail_until_attempt: usize::MAX, + response: "never", + error: "API error (516 ): mapper validation failed: tool schema mismatch", + }), + )], + 5, + 1, + ); + + let result = provider.simple_chat("hello", "test", 0.0).await; + assert!( + result.is_err(), + "516 tool-schema incompatibility should fail quickly without retries" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "tool-schema mismatch must not consume retry budget" + ); + } + + #[tokio::test] + async fn generic_516_without_schema_hint_remains_retryable() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider = ReliableProvider::new( + vec![( + "primary".into(), + Box::new(MockProvider { + calls: Arc::clone(&calls), + fail_until_attempt: 1, + response: "recovered", + error: "API error (516 ): upstream gateway unavailable", + }), + )], + 3, + 1, + ); + + let result = provider.simple_chat("hello", "test", 0.0).await; + assert_eq!(result.unwrap(), "recovered"); + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "generic 516 without schema hint should still retry once and recover" + ); + } + // ── Arc Provider impl for test ── #[async_trait]