Improve agent provider resilience

crates/galaxy_agent_core/src/provider_run.rs

@@ -1010,6 +1010,29 @@ impl ProviderRun {
         Ok(ModelFailureDisposition::RunFailed)
     }
 
+    /// Parks a failed model call so its transcript can be compacted before retrying it.
+    /// The run and current work identity are intentionally unchanged.
+    pub fn prepare_context_compaction(
+        &mut self,
+        work_id: &ExternalWorkId,
+    ) -> Result<(), ProviderRunProtocolError> {
+        let call = match &self.state {
+            ProviderRunState::AwaitingModel { call } => call,
+            ProviderRunState::ReadyToCallModel
+            | ProviderRunState::ResolvingModel { .. }
+            | ProviderRunState::AwaitingTools { .. }
+            | ProviderRunState::AwaitingDriver { .. }
+            | ProviderRunState::Done { .. }
+            | ProviderRunState::Failed { .. }
+            | ProviderRunState::Cancelled { .. } => {
+                return Err(self.unexpected_state(ProviderRunPhase::AwaitingModel));
+            }
+        };
+        validate_work_id(&call.work_id, work_id)?;
+        self.state = ProviderRunState::ReadyToCallModel;
+        Ok(())
+    }
+
     pub fn request_tool_permission(
         &mut self,
         work_id: &ExternalWorkId,

crates/galaxy_agent_core/src/provider_run_tests.rs

@@ -1265,6 +1265,19 @@ fn transcript_compaction_is_rejected_during_a_model_call() {
     ));
 }
 
+#[test]
+fn context_compaction_preserves_run_and_work_identity() {
+    let mut run = run();
+    let call = next_model_call(&mut run);
+    let run_id = run.id().clone();
+
+    run.prepare_context_compaction(&call.work_id).unwrap();
+
+    assert_eq!(run.id(), &run_id);
+    assert_eq!(run.ready_work_id(), Some(call.work_id));
+    assert_eq!(run.state().phase(), ProviderRunPhase::ReadyToCallModel);
+}
+
 #[test]
 fn restore_normalization_commits_a_fully_resolved_batch() {
     let mut run = run();

crates/galaxy_agent_rig/src/stream.rs

@@ -392,10 +392,12 @@ fn completion_error_indicates_recoverable_transport(error: &CompletionError) ->
         // Rig flattens non-status SSE transport failures into ProviderError.
         // Preserve their transport semantics so the durable provider run can
         // retry a truncated or otherwise interrupted response stream.
-        CompletionError::ProviderError(message) => message
-            .to_ascii_lowercase()
-            .starts_with("http client error:"),
-        _ => false,
+        CompletionError::ProviderError(message) => {
+            let normalized = message.to_ascii_lowercase();
+            normalized.starts_with("http client error:")
+                || text_indicates_interrupted_stream(&normalized)
+        }
+        _ => text_indicates_interrupted_stream(&error.to_string().to_ascii_lowercase()),
     }
 }
 
@@ -414,10 +416,29 @@ fn text_indicates_transient_provider_failure(text: &str) -> bool {
         || normalized.contains("service unavailable")
         || normalized.contains("server is overloaded")
         || normalized.contains("servers are currently overloaded")
+        || normalized.contains("overload")
+        || normalized.contains("bad gateway")
+        || normalized.contains("gateway timeout")
+        || normalized.contains("upstream connect error")
+        || normalized.contains("upstream request timeout")
+        || normalized.contains("upstream timed out")
+        || normalized.contains("upstream unavailable")
+}
+
+fn text_indicates_interrupted_stream(text: &str) -> bool {
+    text.contains("unexpected eof")
+        || text.contains("unexpected end of file")
+        || text.contains("stream terminated")
+        || text.contains("stream ended")
+        || text.contains("stream closed")
+        || text.contains("connection reset")
+        || text.contains("connection closed")
+        || text.contains("incomplete message")
 }
 
 fn map_completion_error(error: CompletionError) -> AgentError {
     let is_context_window_exceeded = completion_error_indicates_context_window_exceeded(&error);
+    let is_authentication_failure = completion_error_indicates_authentication_failure(&error);
     let is_recoverable_transport = completion_error_indicates_recoverable_transport(&error);
     let is_transient_provider_failure =
         completion_error_indicates_transient_provider_failure(&error);
@@ -426,7 +447,13 @@ fn map_completion_error(error: CompletionError) -> AgentError {
         .map(|status| status.as_u16());
     let kind = if is_context_window_exceeded {
         AgentErrorKind::ContextWindowExceeded
-    } else if is_recoverable_transport {
+    } else if is_authentication_failure {
+        // Bedrock sometimes returns credential errors as a provider error with no
+        // HTTP status (and Rig can flatten the response body). Classify by the
+        // service error code as well as status so the host can run its existing
+        // credential refresh/login flow.
+        AgentErrorKind::Authentication
+    } else if is_recoverable_transport || matches!(status, Some(408 | 425)) {
         AgentErrorKind::Transport
     } else {
         match status {
@@ -450,14 +477,47 @@ fn map_completion_error(error: CompletionError) -> AgentError {
         }
     };
     let mut mapped = AgentError::new(kind, error.to_string());
+    if is_authentication_failure {
+        mapped.user_message = Some(
+            "AWS credentials for Bedrock are missing or expired. Refresh your AWS login and try again."
+                .to_string(),
+        );
+    }
     mapped.recoverable = matches!(
         kind,
         AgentErrorKind::RateLimited | AgentErrorKind::Transport
-    ) || status.is_some_and(|status| (500..=599).contains(&status))
-        || is_transient_provider_failure;
+    ) || status.is_some_and(|status| {
+        matches!(status, 408 | 425 | 502 | 503 | 504) || (500..=599).contains(&status)
+    }) || is_transient_provider_failure;
     mapped
 }
 
+fn completion_error_indicates_authentication_failure(error: &CompletionError) -> bool {
+    error
+        .provider_response_body()
+        .is_some_and(text_indicates_authentication_failure)
+        || text_indicates_authentication_failure(&error.to_string())
+}
+
+fn text_indicates_authentication_failure(text: &str) -> bool {
+    let normalized = text.to_ascii_lowercase();
+    [
+        "accessdenied",
+        "access denied",
+        "expiredtoken",
+        "expired token",
+        "invalidclienttokenid",
+        "invalid client token",
+        "unrecognizedclient",
+        "unrecognized client",
+        "security token included in the request is invalid",
+        "unable to locate credentials",
+        "credential should be scoped to a valid region",
+    ]
+    .iter()
+    .any(|marker| normalized.contains(marker))
+}
+
 #[cfg(test)]
 mod tests {
     use galaxy_agent_core::{AgentErrorKind, StopReason};
@@ -560,6 +620,20 @@ mod tests {
         assert_eq!(mapped.kind, AgentErrorKind::Provider);
         assert!(mapped.recoverable);
     }
+
+    #[test]
+    fn flattened_bedrock_credential_errors_are_authentication_failures() {
+        for message in [
+            r#"{"__type":"ExpiredTokenException","message":"The security token included in the request is expired"}"#,
+            "UnrecognizedClientException: The security token included in the request is invalid",
+            "AccessDeniedException: not authorized to perform bedrock:ConverseStream",
+        ] {
+            let mapped = map_completion_error(CompletionError::ProviderError(message.to_string()));
+            assert_eq!(mapped.kind, AgentErrorKind::Authentication);
+            assert!(!mapped.recoverable);
+            assert!(mapped.user_message.is_some());
+        }
+    }
 }
 
 #[cfg(test)]

crates/galaxy_agent_rig/src/stream_tests.rs

@@ -27,3 +27,58 @@ fn flattened_streamed_provider_overload_is_recoverable() {
     assert_eq!(mapped.kind, AgentErrorKind::Provider);
     assert!(mapped.recoverable);
 }
+
+#[test]
+fn transient_http_statuses_are_recoverable() {
+    for status_code in [408, 425, 502, 503, 504] {
+        let status = rig_core::http_client::Response::builder()
+            .status(status_code)
+            .body(())
+            .unwrap()
+            .status();
+        let error = CompletionError::from_http_response(status, "transient failure");
+
+        let mapped = map_completion_error(error);
+
+        assert!(mapped.recoverable, "HTTP {status_code} should be retryable");
+        assert_eq!(
+            mapped.kind,
+            if matches!(status_code, 408 | 425) {
+                AgentErrorKind::Transport
+            } else {
+                AgentErrorKind::Provider
+            },
+            "unexpected classification for HTTP {status_code}"
+        );
+    }
+}
+
+#[test]
+fn transient_gateway_upstream_and_overload_messages_are_recoverable() {
+    for message in [
+        "502 Bad Gateway",
+        "upstream request timeout",
+        "upstream timed out",
+        "The server is overloaded; retry later",
+    ] {
+        let mapped = map_completion_error(CompletionError::ProviderError(message.to_string()));
+
+        assert_eq!(mapped.kind, AgentErrorKind::Provider);
+        assert!(mapped.recoverable, "message should be retryable: {message}");
+    }
+}
+
+#[test]
+fn interrupted_stream_messages_are_recoverable_transport() {
+    for message in [
+        "Http client error: unexpected EOF",
+        "Http client error: stream terminated unexpectedly",
+        "Http client error: connection reset by peer",
+        "unexpected EOF",
+    ] {
+        let mapped = map_completion_error(CompletionError::ProviderError(message.to_string()));
+
+        assert_eq!(mapped.kind, AgentErrorKind::Transport);
+        assert!(mapped.recoverable, "message should be retryable: {message}");
+    }
+}

crates/integration/src/test/rig_runtime.rs

@@ -383,7 +383,6 @@ base_url = "http://{address}/v1"
 model_id = "{MODEL_ID}"
 display_name = "Rig Integration Model"
 context_size = 128000
-use_rig = true
 supports_system_messages = false
 "#
     );