mirror of
https://github.com/instructkr/claw-code.git
synced 2026-11-21 00:00:00 +08:00
fix(api): retry 529 overloaded and gateway-flake 400 responses (#3381)
* fix(api): retry 529 + gateway-400 in anthropic provider Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> * fix(api): retry 529 + gateway-400 in openai_compat provider Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> * fix(api): correct EOF whitespace in openai_compat Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --------- Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com>
This commit is contained in:
1 parent
fe2b64ae9e
commit
333298ae53
2 files changed
+188
-7
No files matched your search
@@ -889,7 +889,7 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
|||||||
let request_id = request_id_from_headers(&headers).map(String::into_boxed_str);
|
let request_id = request_id_from_headers(&headers).map(String::into_boxed_str);
|
||||||
let body = response.text().await.unwrap_or_else(|_| String::new());
|
let body = response.text().await.unwrap_or_else(|_| String::new());
|
||||||
let parsed_error = serde_json::from_str::<AnthropicErrorEnvelope>(&body).ok();
|
let parsed_error = serde_json::from_str::<AnthropicErrorEnvelope>(&body).ok();
|
||||||
let retryable = is_retryable_status(status);
|
let retryable = is_retryable_status(status) || is_retryable_400(status, &body);
|
||||||
let retry_after = parse_retry_after(&headers, status).map(Box::new);
|
let retry_after = parse_retry_after(&headers, status).map(Box::new);
|
||||||
|
|
||||||
Err(ApiError::Api {
|
Err(ApiError::Api {
|
||||||
@@ -923,7 +923,10 @@ fn parse_retry_after(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
||||||
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
|
matches!(
|
||||||
|
status.as_u16(),
|
||||||
|
408 | 409 | 429 | 500 | 502 | 503 | 504 | 529
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
||||||
@@ -1146,6 +1149,56 @@ mod tests {
|
|||||||
format!("http://{address}/oauth/token")
|
format!("http://{address}/oauth/token")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn spawn_status_server(status: u16, body: &'static str) -> String {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener");
|
||||||
|
let address = listener.local_addr().expect("local addr");
|
||||||
|
thread::spawn(move || {
|
||||||
|
let (mut stream, _) = listener.accept().expect("accept connection");
|
||||||
|
let mut buffer = [0_u8; 4096];
|
||||||
|
let _ = stream.read(&mut buffer).expect("read request");
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 {status} STATUS\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{}",
|
||||||
|
body.len(),
|
||||||
|
body
|
||||||
|
);
|
||||||
|
stream
|
||||||
|
.write_all(response.as_bytes())
|
||||||
|
.expect("write response");
|
||||||
|
});
|
||||||
|
format!("http://{address}/v1/messages")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expect_success_marks_529_overloaded_retryable() {
|
||||||
|
let url = spawn_status_server(
|
||||||
|
529,
|
||||||
|
r#"{"error":{"type":"overloaded_error","message":"Overloaded"}}"#,
|
||||||
|
);
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("529 should error");
|
||||||
|
assert!(error.is_retryable(), "529 overloaded must be retryable");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expect_success_marks_gateway_flake_400_retryable() {
|
||||||
|
let url = spawn_status_server(400, "HTTP 400 from backend (no parseable body)");
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("400 should error");
|
||||||
|
assert!(
|
||||||
|
error.is_retryable(),
|
||||||
|
"gateway-flake 400 body must be retryable"
|
||||||
|
);
|
||||||
|
let url = spawn_status_server(400, "invalid request: max_tokens must be positive");
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("400 should error");
|
||||||
|
assert!(!error.is_retryable(), "a real 400 must not be retried");
|
||||||
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn read_api_key_requires_presence() {
|
fn read_api_key_requires_presence() {
|
||||||
let _guard = env_lock();
|
let _guard = env_lock();
|
||||||
@@ -1445,6 +1498,40 @@ mod tests {
|
|||||||
assert!(!super::is_retryable_status(
|
assert!(!super::is_retryable_status(
|
||||||
reqwest::StatusCode::UNAUTHORIZED
|
reqwest::StatusCode::UNAUTHORIZED
|
||||||
));
|
));
|
||||||
|
// Anthropic returns the non-standard 529 when the API is overloaded;
|
||||||
|
// it is transient and must be retried like a 503.
|
||||||
|
assert!(super::is_retryable_status(
|
||||||
|
reqwest::StatusCode::from_u16(529).expect("529 is a valid status")
|
||||||
|
));
|
||||||
|
assert!(!super::is_retryable_status(
|
||||||
|
reqwest::StatusCode::BAD_REQUEST
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_flake_400_bodies_are_retryable() {
|
||||||
|
// Gateways and proxies surface transient network failures as HTTP 400
|
||||||
|
// with an unparseable body; is_retryable_400 detects them so
|
||||||
|
// expect_success marks the resulting error retryable.
|
||||||
|
for body in [
|
||||||
|
"HTTP 400 from backend (no parseable body)",
|
||||||
|
"upstream connection reset by peer",
|
||||||
|
"Broken pipe while reading response",
|
||||||
|
"Empty reply from server",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
super::is_retryable_400(reqwest::StatusCode::BAD_REQUEST, body),
|
||||||
|
"body should be retryable: {body}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!super::is_retryable_400(
|
||||||
|
reqwest::StatusCode::BAD_REQUEST,
|
||||||
|
"invalid request: max_tokens must be positive"
|
||||||
|
));
|
||||||
|
assert!(!super::is_retryable_400(
|
||||||
|
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"connection reset"
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1763,7 +1763,7 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
|||||||
let request_id = request_id_from_headers(&headers).map(String::into_boxed_str);
|
let request_id = request_id_from_headers(&headers).map(String::into_boxed_str);
|
||||||
let body = response.text().await.unwrap_or_default();
|
let body = response.text().await.unwrap_or_default();
|
||||||
let parsed_error = serde_json::from_str::<ErrorEnvelope>(&body).ok();
|
let parsed_error = serde_json::from_str::<ErrorEnvelope>(&body).ok();
|
||||||
let retryable = is_retryable_status(status);
|
let retryable = is_retryable_status(status) || is_retryable_400(status, &body);
|
||||||
let retry_after = parse_retry_after(&headers, status).map(Box::new);
|
let retry_after = parse_retry_after(&headers, status).map(Box::new);
|
||||||
|
|
||||||
let suggested_action = suggested_action_for_status(status);
|
let suggested_action = suggested_action_for_status(status);
|
||||||
@@ -1799,7 +1799,10 @@ fn parse_retry_after(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
||||||
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
|
matches!(
|
||||||
|
status.as_u16(),
|
||||||
|
408 | 409 | 429 | 500 | 502 | 503 | 504 | 529
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
||||||
@@ -1858,9 +1861,9 @@ impl StringExt for String {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
build_chat_completion_request, chat_completions_endpoint, is_reasoning_model,
|
build_chat_completion_request, chat_completions_endpoint, is_reasoning_model,
|
||||||
model_requires_reasoning_content_in_history, normalize_finish_reason, normalize_response,
|
is_retryable_400, is_retryable_status, model_requires_reasoning_content_in_history,
|
||||||
openai_tool_choice, parse_tool_arguments, OpenAiCompatClient, OpenAiCompatConfig,
|
normalize_finish_reason, normalize_response, openai_tool_choice, parse_tool_arguments,
|
||||||
StreamState,
|
OpenAiCompatClient, OpenAiCompatConfig, StreamState,
|
||||||
};
|
};
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::test_env_lock as env_lock;
|
use crate::test_env_lock as env_lock;
|
||||||
@@ -1872,6 +1875,97 @@ mod tests {
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
fn spawn_status_server(status: u16, body: &'static str) -> String {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener");
|
||||||
|
let address = listener.local_addr().expect("local addr");
|
||||||
|
thread::spawn(move || {
|
||||||
|
let (mut stream, _) = listener.accept().expect("accept connection");
|
||||||
|
let mut buffer = [0_u8; 4096];
|
||||||
|
let _ = stream.read(&mut buffer).expect("read request");
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 {status} STATUS\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{}",
|
||||||
|
body.len(),
|
||||||
|
body
|
||||||
|
);
|
||||||
|
stream
|
||||||
|
.write_all(response.as_bytes())
|
||||||
|
.expect("write response");
|
||||||
|
});
|
||||||
|
format!("http://{address}/v1/messages")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expect_success_marks_529_overloaded_retryable() {
|
||||||
|
let url = spawn_status_server(
|
||||||
|
529,
|
||||||
|
r#"{"error":{"type":"overloaded_error","message":"Overloaded"}}"#,
|
||||||
|
);
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("529 should error");
|
||||||
|
assert!(error.is_retryable(), "529 overloaded must be retryable");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expect_success_marks_gateway_flake_400_retryable() {
|
||||||
|
let url = spawn_status_server(400, "HTTP 400 from backend (no parseable body)");
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("400 should error");
|
||||||
|
assert!(
|
||||||
|
error.is_retryable(),
|
||||||
|
"gateway-flake 400 body must be retryable"
|
||||||
|
);
|
||||||
|
let url = spawn_status_server(400, "invalid request: max_tokens must be positive");
|
||||||
|
let response = reqwest::get(&url).await.expect("response");
|
||||||
|
let error = super::expect_success(response)
|
||||||
|
.await
|
||||||
|
.expect_err("400 should error");
|
||||||
|
assert!(!error.is_retryable(), "a real 400 must not be retried");
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn retryable_statuses_are_detected() {
|
||||||
|
assert!(is_retryable_status(reqwest::StatusCode::TOO_MANY_REQUESTS));
|
||||||
|
assert!(is_retryable_status(
|
||||||
|
reqwest::StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
));
|
||||||
|
assert!(!is_retryable_status(reqwest::StatusCode::UNAUTHORIZED));
|
||||||
|
// Several OpenAI-compatible gateways (and Anthropic) signal overload
|
||||||
|
// with the non-standard 529; it is transient and must be retried.
|
||||||
|
assert!(is_retryable_status(
|
||||||
|
reqwest::StatusCode::from_u16(529).expect("529 is a valid status")
|
||||||
|
));
|
||||||
|
assert!(!is_retryable_status(reqwest::StatusCode::BAD_REQUEST));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_flake_400_bodies_are_retryable() {
|
||||||
|
for body in [
|
||||||
|
"HTTP 400 from backend (no parseable body)",
|
||||||
|
"upstream connection reset by peer",
|
||||||
|
"Broken pipe while reading response",
|
||||||
|
"Empty reply from server",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
is_retryable_400(reqwest::StatusCode::BAD_REQUEST, body),
|
||||||
|
"body should be retryable: {body}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!is_retryable_400(
|
||||||
|
reqwest::StatusCode::BAD_REQUEST,
|
||||||
|
"invalid request: max_tokens must be positive"
|
||||||
|
));
|
||||||
|
assert!(!is_retryable_400(
|
||||||
|
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"connection reset"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn request_translation_uses_openai_compatible_shape() {
|
fn request_translation_uses_openai_compatible_shape() {
|
||||||
|
|||||||
Reference in new issue
Block a user