diff --git a/crates/tinyinference-llm/src/providers/openai/test.rs b/crates/tinyinference-llm/src/providers/openai/test.rs index 93013d20..9bc68946 100644 --- a/crates/tinyinference-llm/src/providers/openai/test.rs +++ b/crates/tinyinference-llm/src/providers/openai/test.rs @@ -583,6 +583,7 @@ fn parse_error_body_classifies_retryability_by_http_status() { let unauthorized = m.parse_error_body( 401, r#"{"error":{"message":"Incorrect API key provided","type":"invalid_request_error","code":"invalid_api_key"}}"#, + None, ); assert_eq!(unauthorized.status, Some(401)); assert!(!unauthorized.retryable, "401 must not be retryable"); @@ -590,14 +591,39 @@ fn parse_error_body_classifies_retryability_by_http_status() { let rate_limited = m.parse_error_body( 429, r#"{"error":{"message":"Rate limit reached","type":"requests","code":"rate_limit_exceeded"}}"#, + None, ); assert_eq!(rate_limited.status, Some(429)); assert!(rate_limited.retryable, "429 must be retryable"); - let server_error = m.parse_error_body(500, r#"{"error":{"message":"internal error"}}"#); + let server_error = m.parse_error_body(500, r#"{"error":{"message":"internal error"}}"#, None); assert!(server_error.retryable, "5xx must be retryable"); } +#[test] +fn parse_error_body_carries_retry_after_into_the_structured_error() { + // A 429 usually says how long to wait. The retry layer prefers + // `retry_after_ms` over its own backoff curve, and the Anthropic + // transport already fills it from the same header; this transport used + // to hardcode `None`, so a server-supplied wait was never honoured on + // OpenAI-compatible endpoints (including the managed backend). + let m = model(); + let body = r#"{"error":{"message":"Upstream rate limit exceeded for model 'x'. Please retry shortly.","type":"rate_limit_error"}}"#; + + let with_header = m.parse_error_body(429, body, Some("7")); + assert_eq!(with_header.status, Some(429)); + assert!(with_header.retryable); + assert_eq!(with_header.retry_after_ms, Some(7_000)); + + // Without the header there is still no number in the message to fall + // back on, so the field stays empty rather than guessing. + let without_header = m.parse_error_body(429, body, None); + assert_eq!(without_header.retry_after_ms, None); + + let unparseable = m.parse_error_body(429, body, Some("soon")); + assert_eq!(unparseable.retry_after_ms, None); +} + #[test] fn reasoning_tag_extraction_defaults_off_for_hosted_openai_only() { // Hosted OpenAI never emits inline `` reasoning; unconditional diff --git a/crates/tinyinference-llm/src/providers/openai/transport.rs b/crates/tinyinference-llm/src/providers/openai/transport.rs index cf177892..a1970355 100644 --- a/crates/tinyinference-llm/src/providers/openai/transport.rs +++ b/crates/tinyinference-llm/src/providers/openai/transport.rs @@ -1491,15 +1491,27 @@ impl OpenAiModel { url: &str, ) -> Result { let response = builder.send().await.map_err(|e| { - let error = - self.provider_error(format!("{what} to {url} failed: {e}"), None, None, None); + let error = self.provider_error( + format!("{what} to {url} failed: {e}"), + None, + None, + None, + None, + ); Error::Provider(Box::new(error)) })?; let status = response.status(); if !status.is_success() { + // Read the header before `text()` consumes the response: a 429's + // `Retry-After` is the one wait the retry layer cannot guess. + let retry_after = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); let text = response.text().await.unwrap_or_default(); - let error = self.parse_error_body(status.as_u16(), &text); + let error = self.parse_error_body(status.as_u16(), &text, retry_after.as_deref()); return Err(Error::Provider(Box::new(error))); } Ok(response) @@ -1595,6 +1607,7 @@ impl OpenAiModel { status: Option, code: Option, raw: Option, + retry_after: Option<&str>, ) -> ProviderError { let message = message.into(); let retryable = @@ -1608,13 +1621,18 @@ impl OpenAiModel { message, retryable, raw, - retry_after_ms: None, + retry_after_ms: tinyinference_core::parse_retry_after_ms(retry_after), partial_message: None, stop_reason: None, } } - pub(super) fn parse_error_body(&self, status: u16, text: &str) -> ProviderError { + pub(super) fn parse_error_body( + &self, + status: u16, + text: &str, + retry_after: Option<&str>, + ) -> ProviderError { let raw = serde_json::from_str::(text).ok(); let error_obj = raw.as_ref().and_then(|value| value.get("error")); let message = error_obj @@ -1643,7 +1661,7 @@ impl OpenAiModel { missing_model_remediation(kind, status, &message, &self.model, &self.base_url) }) .unwrap_or(message); - self.provider_error(message, Some(status), code, raw) + self.provider_error(message, Some(status), code, raw, retry_after) } } @@ -2077,7 +2095,7 @@ fn responses_sse_failure(body: &str, model: &OpenAiModel) -> Option