mirror of
https://github.com/instructkr/claw-code.git
synced 2026-04-07 00:24:50 +08:00
Generic fatal wrapper handling already preserved safe classes and trace ids for single provider failures, but repeated retry exhaustion still surfaced as provider_internal. Classify generic wrapped RetriesExhausted failures as provider_retry_exhausted so Jobdori-style repeat failures stay distinguishable from one-off provider crashes, and keep the display logic clippy-clean. Constraint: Keep the change minimal and preserve existing user-visible error wording outside retry-exhaustion classification Rejected: Broadly rework all provider error taxonomy | unnecessary for the targeted opaque-wrapper regression Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep retry exhaustion distinct from single-shot provider_internal wrappers when the nested error is the same generic fatal wrapper Tested: cargo test -p api detects_generic_fatal_wrapper_and_classifies_it_as_provider_internal Tested: cargo test -p api retries_exhausted_preserves_nested_request_id_and_failure_class Tested: cargo test -p rusty-claude-cli opaque_provider_wrapper_surfaces_failure_class_session_and_trace Tested: cargo test -p rusty-claude-cli retry_exhaustion_uses_retry_failure_class_for_generic_provider_wrapper Tested: cargo test --workspace Tested: cargo fmt --check Tested: cargo clippy --workspace --all-targets -- -D warnings Not-tested: Live OpenClaw/Anthropic service failure telemetry outside the local test harness
354 lines
12 KiB
Rust
354 lines
12 KiB
Rust
use std::env::VarError;
|
|
use std::fmt::{Display, Formatter};
|
|
use std::time::Duration;
|
|
|
|
const GENERIC_FATAL_WRAPPER_MARKERS: &[&str] = &[
|
|
"something went wrong while processing your request",
|
|
"please try again, or use /new to start a fresh session",
|
|
];
|
|
|
|
const CONTEXT_WINDOW_ERROR_MARKERS: &[&str] = &[
|
|
"maximum context length",
|
|
"context window",
|
|
"context length",
|
|
"too many tokens",
|
|
"prompt is too long",
|
|
"input is too long",
|
|
"request is too large",
|
|
];
|
|
|
|
#[derive(Debug)]
|
|
pub enum ApiError {
|
|
MissingCredentials {
|
|
provider: &'static str,
|
|
env_vars: &'static [&'static str],
|
|
},
|
|
ContextWindowExceeded {
|
|
model: String,
|
|
estimated_input_tokens: u32,
|
|
requested_output_tokens: u32,
|
|
estimated_total_tokens: u32,
|
|
context_window_tokens: u32,
|
|
},
|
|
ExpiredOAuthToken,
|
|
Auth(String),
|
|
InvalidApiKeyEnv(VarError),
|
|
Http(reqwest::Error),
|
|
Io(std::io::Error),
|
|
Json(serde_json::Error),
|
|
Api {
|
|
status: reqwest::StatusCode,
|
|
error_type: Option<String>,
|
|
message: Option<String>,
|
|
request_id: Option<String>,
|
|
body: String,
|
|
retryable: bool,
|
|
},
|
|
RetriesExhausted {
|
|
attempts: u32,
|
|
last_error: Box<ApiError>,
|
|
},
|
|
InvalidSseFrame(&'static str),
|
|
BackoffOverflow {
|
|
attempt: u32,
|
|
base_delay: Duration,
|
|
},
|
|
}
|
|
|
|
impl ApiError {
|
|
#[must_use]
|
|
pub const fn missing_credentials(
|
|
provider: &'static str,
|
|
env_vars: &'static [&'static str],
|
|
) -> Self {
|
|
Self::MissingCredentials { provider, env_vars }
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn is_retryable(&self) -> bool {
|
|
match self {
|
|
Self::Http(error) => error.is_connect() || error.is_timeout() || error.is_request(),
|
|
Self::Api { retryable, .. } => *retryable,
|
|
Self::RetriesExhausted { last_error, .. } => last_error.is_retryable(),
|
|
Self::MissingCredentials { .. }
|
|
| Self::ContextWindowExceeded { .. }
|
|
| Self::ExpiredOAuthToken
|
|
| Self::Auth(_)
|
|
| Self::InvalidApiKeyEnv(_)
|
|
| Self::Io(_)
|
|
| Self::Json(_)
|
|
| Self::InvalidSseFrame(_)
|
|
| Self::BackoffOverflow { .. } => false,
|
|
}
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn request_id(&self) -> Option<&str> {
|
|
match self {
|
|
Self::Api { request_id, .. } => request_id.as_deref(),
|
|
Self::RetriesExhausted { last_error, .. } => last_error.request_id(),
|
|
Self::MissingCredentials { .. }
|
|
| Self::ContextWindowExceeded { .. }
|
|
| Self::ExpiredOAuthToken
|
|
| Self::Auth(_)
|
|
| Self::InvalidApiKeyEnv(_)
|
|
| Self::Http(_)
|
|
| Self::Io(_)
|
|
| Self::Json(_)
|
|
| Self::InvalidSseFrame(_)
|
|
| Self::BackoffOverflow { .. } => None,
|
|
}
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn safe_failure_class(&self) -> &'static str {
|
|
match self {
|
|
Self::RetriesExhausted { .. } if self.is_context_window_failure() => "context_window",
|
|
Self::RetriesExhausted { .. } if self.is_generic_fatal_wrapper() => {
|
|
"provider_retry_exhausted"
|
|
}
|
|
Self::RetriesExhausted { last_error, .. } => last_error.safe_failure_class(),
|
|
Self::MissingCredentials { .. } | Self::ExpiredOAuthToken | Self::Auth(_) => {
|
|
"provider_auth"
|
|
}
|
|
Self::Api { status, .. } if matches!(status.as_u16(), 401 | 403) => "provider_auth",
|
|
Self::ContextWindowExceeded { .. } => "context_window",
|
|
Self::Api { .. } if self.is_context_window_failure() => "context_window",
|
|
Self::Api { status, .. } if status.as_u16() == 429 => "provider_rate_limit",
|
|
Self::Api { .. } if self.is_generic_fatal_wrapper() => "provider_internal",
|
|
Self::Api { .. } => "provider_error",
|
|
Self::Http(_) | Self::InvalidSseFrame(_) | Self::BackoffOverflow { .. } => {
|
|
"provider_transport"
|
|
}
|
|
Self::InvalidApiKeyEnv(_) | Self::Io(_) | Self::Json(_) => "runtime_io",
|
|
}
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn is_generic_fatal_wrapper(&self) -> bool {
|
|
match self {
|
|
Self::Api { message, body, .. } => {
|
|
message
|
|
.as_deref()
|
|
.is_some_and(looks_like_generic_fatal_wrapper)
|
|
|| looks_like_generic_fatal_wrapper(body)
|
|
}
|
|
Self::RetriesExhausted { last_error, .. } => last_error.is_generic_fatal_wrapper(),
|
|
Self::MissingCredentials { .. }
|
|
| Self::ContextWindowExceeded { .. }
|
|
| Self::ExpiredOAuthToken
|
|
| Self::Auth(_)
|
|
| Self::InvalidApiKeyEnv(_)
|
|
| Self::Http(_)
|
|
| Self::Io(_)
|
|
| Self::Json(_)
|
|
| Self::InvalidSseFrame(_)
|
|
| Self::BackoffOverflow { .. } => false,
|
|
}
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn is_context_window_failure(&self) -> bool {
|
|
match self {
|
|
Self::ContextWindowExceeded { .. } => true,
|
|
Self::Api {
|
|
status,
|
|
message,
|
|
body,
|
|
..
|
|
} => {
|
|
matches!(status.as_u16(), 400 | 413 | 422)
|
|
&& (message
|
|
.as_deref()
|
|
.is_some_and(looks_like_context_window_error)
|
|
|| looks_like_context_window_error(body))
|
|
}
|
|
Self::RetriesExhausted { last_error, .. } => last_error.is_context_window_failure(),
|
|
Self::MissingCredentials { .. }
|
|
| Self::ExpiredOAuthToken
|
|
| Self::Auth(_)
|
|
| Self::InvalidApiKeyEnv(_)
|
|
| Self::Http(_)
|
|
| Self::Io(_)
|
|
| Self::Json(_)
|
|
| Self::InvalidSseFrame(_)
|
|
| Self::BackoffOverflow { .. } => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Display for ApiError {
|
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
match self {
|
|
Self::MissingCredentials { provider, env_vars } => write!(
|
|
f,
|
|
"missing {provider} credentials; export {} before calling the {provider} API",
|
|
env_vars.join(" or ")
|
|
),
|
|
Self::ContextWindowExceeded {
|
|
model,
|
|
estimated_input_tokens,
|
|
requested_output_tokens,
|
|
estimated_total_tokens,
|
|
context_window_tokens,
|
|
} => write!(
|
|
f,
|
|
"context_window_blocked for {model}: estimated input {estimated_input_tokens} + requested output {requested_output_tokens} = {estimated_total_tokens} tokens exceeds the {context_window_tokens}-token context window; compact the session or reduce request size before retrying"
|
|
),
|
|
Self::ExpiredOAuthToken => {
|
|
write!(
|
|
f,
|
|
"saved OAuth token is expired and no refresh token is available"
|
|
)
|
|
}
|
|
Self::Auth(message) => write!(f, "auth error: {message}"),
|
|
Self::InvalidApiKeyEnv(error) => {
|
|
write!(f, "failed to read credential environment variable: {error}")
|
|
}
|
|
Self::Http(error) => write!(f, "http error: {error}"),
|
|
Self::Io(error) => write!(f, "io error: {error}"),
|
|
Self::Json(error) => write!(f, "json error: {error}"),
|
|
Self::Api {
|
|
status,
|
|
error_type,
|
|
message,
|
|
request_id,
|
|
body,
|
|
..
|
|
} => {
|
|
if let (Some(error_type), Some(message)) = (error_type, message) {
|
|
write!(f, "api returned {status} ({error_type})")?;
|
|
if let Some(request_id) = request_id {
|
|
write!(f, " [trace {request_id}]")?;
|
|
}
|
|
write!(f, ": {message}")
|
|
} else {
|
|
write!(f, "api returned {status}")?;
|
|
if let Some(request_id) = request_id {
|
|
write!(f, " [trace {request_id}]")?;
|
|
}
|
|
write!(f, ": {body}")
|
|
}
|
|
}
|
|
Self::RetriesExhausted {
|
|
attempts,
|
|
last_error,
|
|
} => write!(f, "api failed after {attempts} attempts: {last_error}"),
|
|
Self::InvalidSseFrame(message) => write!(f, "invalid sse frame: {message}"),
|
|
Self::BackoffOverflow {
|
|
attempt,
|
|
base_delay,
|
|
} => write!(
|
|
f,
|
|
"retry backoff overflowed on attempt {attempt} with base delay {base_delay:?}"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for ApiError {}
|
|
|
|
impl From<reqwest::Error> for ApiError {
|
|
fn from(value: reqwest::Error) -> Self {
|
|
Self::Http(value)
|
|
}
|
|
}
|
|
|
|
impl From<std::io::Error> for ApiError {
|
|
fn from(value: std::io::Error) -> Self {
|
|
Self::Io(value)
|
|
}
|
|
}
|
|
|
|
impl From<serde_json::Error> for ApiError {
|
|
fn from(value: serde_json::Error) -> Self {
|
|
Self::Json(value)
|
|
}
|
|
}
|
|
|
|
impl From<VarError> for ApiError {
|
|
fn from(value: VarError) -> Self {
|
|
Self::InvalidApiKeyEnv(value)
|
|
}
|
|
}
|
|
|
|
fn looks_like_generic_fatal_wrapper(text: &str) -> bool {
|
|
let lowered = text.to_ascii_lowercase();
|
|
GENERIC_FATAL_WRAPPER_MARKERS
|
|
.iter()
|
|
.any(|marker| lowered.contains(marker))
|
|
}
|
|
|
|
fn looks_like_context_window_error(text: &str) -> bool {
|
|
let lowered = text.to_ascii_lowercase();
|
|
CONTEXT_WINDOW_ERROR_MARKERS
|
|
.iter()
|
|
.any(|marker| lowered.contains(marker))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::ApiError;
|
|
|
|
#[test]
|
|
fn detects_generic_fatal_wrapper_and_classifies_it_as_provider_internal() {
|
|
let error = ApiError::Api {
|
|
status: reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
|
error_type: Some("api_error".to_string()),
|
|
message: Some(
|
|
"Something went wrong while processing your request. Please try again, or use /new to start a fresh session."
|
|
.to_string(),
|
|
),
|
|
request_id: Some("req_jobdori_123".to_string()),
|
|
body: String::new(),
|
|
retryable: true,
|
|
};
|
|
|
|
assert!(error.is_generic_fatal_wrapper());
|
|
assert_eq!(error.safe_failure_class(), "provider_internal");
|
|
assert_eq!(error.request_id(), Some("req_jobdori_123"));
|
|
assert!(error.to_string().contains("[trace req_jobdori_123]"));
|
|
}
|
|
|
|
#[test]
|
|
fn retries_exhausted_preserves_nested_request_id_and_failure_class() {
|
|
let error = ApiError::RetriesExhausted {
|
|
attempts: 3,
|
|
last_error: Box::new(ApiError::Api {
|
|
status: reqwest::StatusCode::BAD_GATEWAY,
|
|
error_type: Some("api_error".to_string()),
|
|
message: Some(
|
|
"Something went wrong while processing your request. Please try again, or use /new to start a fresh session."
|
|
.to_string(),
|
|
),
|
|
request_id: Some("req_nested_456".to_string()),
|
|
body: String::new(),
|
|
retryable: true,
|
|
}),
|
|
};
|
|
|
|
assert!(error.is_generic_fatal_wrapper());
|
|
assert_eq!(error.safe_failure_class(), "provider_retry_exhausted");
|
|
assert_eq!(error.request_id(), Some("req_nested_456"));
|
|
}
|
|
|
|
#[test]
|
|
fn classifies_provider_context_window_errors() {
|
|
let error = ApiError::Api {
|
|
status: reqwest::StatusCode::BAD_REQUEST,
|
|
error_type: Some("invalid_request_error".to_string()),
|
|
message: Some(
|
|
"This model's maximum context length is 200000 tokens, but your request used 230000 tokens."
|
|
.to_string(),
|
|
),
|
|
request_id: Some("req_ctx_123".to_string()),
|
|
body: String::new(),
|
|
retryable: false,
|
|
};
|
|
|
|
assert!(error.is_context_window_failure());
|
|
assert_eq!(error.safe_failure_class(), "context_window");
|
|
assert_eq!(error.request_id(), Some("req_ctx_123"));
|
|
}
|
|
}
|