public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first pass of merging in warp (doesn't build)

Browse Source
This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/ai/blocklist/controller/response_stream.rs
+305 -142
View File
@@ -1,4 +1,6 @@
use std::{cell::RefCell, rc::Rc};
use std::cell::RefCell;
use std::rc::Rc;
use std::sync::Arc;
use anyhow::anyhow;
use chrono::{DateTime, Local, TimeDelta};
@@ -7,22 +9,54 @@ use galaxyui::{Entity, ModelContext, SingletonEntity};
use uuid::Uuid;
use warp_multi_agent_api::response_event;
use crate::{
ai::{
agent::{
api::{self, generate_multi_agent_output, ConvertToAPITypeError},
conversation::AIConversationId,
AIIdentifiers, CancellationReason,
},
bedrock::client::BedrockClientConfig,
openai::client::OpenAIClientConfig,
provider::ProviderConfig,
},
network::NetworkStatus,
report_error, send_telemetry_from_ctx,
settings::ai::AISettings,
};
use settings::Setting;
use crate::ai::agent::api::{self, generate_multi_agent_output, ConvertToAPITypeError};
use crate::ai::agent::conversation::AIConversationId;
use crate::ai::agent::{AIIdentifiers, CancellationReason};
use crate::network::NetworkStatus;
use crate::server::server_api::{AIApiError, ServerApiProvider};
use crate::{report_error, send_telemetry_from_ctx};
/// Maximum number of times a single MAA request is re-sent before the failure is
/// surfaced.
const MAX_RETRIES: usize = 3;
/// What to do about a failed or truncated MAA response attempt.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RecoveryAction {
/// Re-send the same request immediately.
RetryNow,
/// Re-send the same request once connectivity returns.
RetryWhenOnline,
/// Resume the conversation with a fresh request after the stream completes.
Resume,
/// Surface the error; the conversation ends in error.
Fail,
}
/// Decides how to recover from a failed response-stream attempt.
///
/// Before any client actions have been received, the request can be re-sent verbatim
/// (immediately, or once connectivity returns). After actions have streamed,
/// re-sending is unsafe, so recovery uses a fresh `ResumeConversation` request.
fn recovery_action(
has_received_client_actions: bool,
is_recoverable: bool,
has_retry_budget: bool,
can_attempt_resume_on_error: bool,
is_online: bool,
) -> RecoveryAction {
if !has_received_client_actions && is_recoverable && has_retry_budget {
if is_online {
RecoveryAction::RetryNow
} else {
RecoveryAction::RetryWhenOnline
}
} else if has_received_client_actions && is_recoverable && can_attempt_resume_on_error {
RecoveryAction::Resume
} else {
RecoveryAction::Fail
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ResponseStreamId(String);
@@ -70,10 +104,23 @@ pub struct ResponseStream {
/// Whether we should attempt to resume the conversation after the stream finishes.
///
/// This is set when we receive a retryable error after client actions have been received
/// and `can_attempt_resume_on_error` is true.
/// This is set when a transient network/server failure occurs after client actions
/// have been received (so an in-request retry is unsafe) and
/// `can_attempt_resume_on_error` is true.
should_resume_conversation_after_stream_finished: bool,
/// Whether a `StreamFinished` event was received for the current request. A
/// stream that completes without one was truncated in transit.
stream_finished_received: bool,
/// Whether a terminal error event has already been emitted for the current
/// request, so stream completion doesn't synthesize a second failure for it.
error_event_emitted: bool,
/// Whether a retry is parked waiting for connectivity. While set, completion of
/// the failed attempt's underlying stream is ignored.
deferred_retry_pending: bool,
/// Unique, internal id for the current request.
///
/// This ensures that the model never emits events for a request that was already cancelled (or
@@ -85,65 +132,37 @@ pub struct ResponseStream {
}
impl ResponseStream {
fn resolve_provider_config(model_id: &str, ctx: &ModelContext<Self>) -> ProviderConfig {
let settings = AISettings::as_ref(ctx);
// Check if OpenAI/LiteLLM provider is enabled
if *settings.openai_enabled.value() {
// First, check if this specific model has a per-provider routing entry
// (from the multi-provider `ai.providers[]` config or legacy `ai.openai.models`).
use crate::ai::llms::LLMPreferences;
let prefs = LLMPreferences::as_ref(ctx);
if let Some(config) = prefs.openai_client_config_for_model(model_id) {
return ProviderConfig::OpenAI(OpenAIClientConfig {
base_url: config.base_url.clone(),
api_key: config.api_key.clone(),
model: Some(model_id.to_string()),
});
}
// Fall back to the legacy single-provider config
let base_url = settings.openai_base_url.value().clone();
let api_key = {
let key = settings.openai_api_key.value().clone();
if key.is_empty() {
None
} else {
Some(key)
}
};
let model = {
let m = settings.openai_model.value().clone();
if m.is_empty() {
Some(model_id.to_string())
} else {
Some(m)
}
};
return ProviderConfig::OpenAI(OpenAIClientConfig {
base_url,
api_key,
model,
});
/// Emits a synthetic successful response event through the normal controller subscription.
#[cfg(test)]
pub fn emit_response_event_for_test(
&mut self,
event: warp_multi_agent_api::ResponseEvent,
ctx: &mut ModelContext<Self>,
) {
ctx.emit(ResponseStreamEvent::ReceivedEvent(Consumable::new(Ok(
event,
))));
}
#[cfg(test)]
pub fn new_for_test(id: ResponseStreamId) -> Self {
let (cancellation_tx, _rx) = oneshot::channel();
Self {
id,
params: api::RequestParams::new_for_test(),
retry_count: 0,
start_time: Local::now(),
time_to_latest_event: TimeDelta::seconds(0),
cancellation_tx: Some(cancellation_tx),
original_error: None,
has_received_client_actions: false,
ai_identifiers: AIIdentifiers::default(),
can_attempt_resume_on_error: false,
should_resume_conversation_after_stream_finished: false,
stream_finished_received: false,
error_event_emitted: false,
deferred_retry_pending: false,
current_request_id: Some(Uuid::new_v4()),
}
// Fall back to Bedrock
if *settings.bedrock_enabled.value() {
let auth_method = *settings.bedrock_auth_method.value();
return ProviderConfig::Bedrock(
BedrockClientConfig {
auth_method,
profile: settings.bedrock_profile.value().clone(),
region: settings.bedrock_region.value().clone(),
access_key_id: settings.bedrock_access_key_id.value().clone(),
secret_access_key: settings.bedrock_secret_access_key.value().clone(),
cross_region_inference: *settings.bedrock_cross_region_inference.value(),
}
.with_external_fallbacks(),
);
}
ProviderConfig::None
}
pub fn new(
@@ -178,6 +197,9 @@ impl ResponseStream {
ai_identifiers,
can_attempt_resume_on_error,
should_resume_conversation_after_stream_finished: false,
stream_finished_received: false,
error_event_emitted: false,
deferred_retry_pending: false,
current_request_id: Some(request_id),
}
}
@@ -222,7 +244,11 @@ impl ResponseStream {
fn retry(&mut self, ctx: &mut ModelContext<Self>) {
self.retry_count += 1;
self.has_received_client_actions = false; // Reset for the new attempt
// Reset per-attempt state for the new attempt.
self.has_received_client_actions = false;
self.stream_finished_received = false;
self.error_event_emitted = false;
self.deferred_retry_pending = false;
let (cancellation_tx, cancellation_rx) = oneshot::channel();
if let Some(old_cancellation_tx) = self.cancellation_tx.take() {
@@ -285,6 +311,21 @@ impl ResponseStream {
}
Err(e) => {
log::error!("Failed to send request to multi-agent API: {e:?}");
if self.current_request_id.is_none_or(|id| id != request_id) {
return;
}
// A request-conversion failure is a deterministic client-side error and
// no stream was ever created: retrying would fail identically, and
// letting completion synthesize `UnexpectedEof` would misreport it as
// a transient network failure. Surface the original error and finish
// terminally. (HTTP send failures don't take this path — they arrive as
// in-stream error events.)
let error = Arc::new(AIApiError::Other(anyhow!(e)));
self.error_event_emitted = true;
self.report_request_failure(&error, NetworkStatus::as_ref(ctx).is_online());
ctx.emit(ResponseStreamEvent::ReceivedEvent(Consumable::new(Err(
error,
))));
self.on_response_stream_complete(request_id, ctx);
}
}
@@ -342,6 +383,7 @@ impl ResponseStream {
self.has_received_client_actions = true;
}
warp_multi_agent_api::response_event::Type::Finished(finished_event) => {
self.stream_finished_received = true;
// Emit retry success telemetry on successful completion
if matches!(
finished_event.reason,
@@ -373,72 +415,53 @@ impl ResponseStream {
self.original_error = Some(format!("{e:?}"));
}
// Only retry if:
// 1. We haven't received any client actions yet (this is the first event or only init events)
// 2. The error is retryable
// 3. We haven't exceeded max retries
// 4. We're online
const MAX_RETRIES: usize = 3;
let network_status = NetworkStatus::as_ref(ctx);
let is_online = network_status.is_online();
let is_retryable = e.is_retryable();
let should_retry = !self.has_received_client_actions
&& is_retryable
&& self.retry_count < MAX_RETRIES
&& is_online;
if should_retry {
log::warn!(
"MultiAgent request failed, retrying (attempt {}/{}) - Error: {e:?}",
self.retry_count + 1,
MAX_RETRIES
);
// Only emit error telemetry here if we're retrying.
// Final errors that aren't being retried are emitted elsewhere.
self.emit_retryable_agent_mode_error_telemetry(format!("{e:?}"), ctx);
self.retry(ctx);
// Don't emit the error event, we're retrying
// TODO: emit a separate event if controller needs to know about failures that are being retried
return;
}
// If we can't retry (because client actions were received) but the error is
// retryable and we're allowed to attempt a resume, signal that the controller
// should resume the conversation after the stream completes.
let should_attempt_resume = self.has_received_client_actions
&& is_retryable
&& self.can_attempt_resume_on_error;
if should_attempt_resume {
self.should_resume_conversation_after_stream_finished = true;
}
#[cfg(feature = "crash_reporting")]
sentry::with_scope(
|scope| {
scope.set_tag(
"has_received_client_actions",
self.has_received_client_actions,
let is_online = NetworkStatus::as_ref(ctx).is_online();
match recovery_action(
self.has_received_client_actions,
e.is_recoverable(),
self.retry_count < MAX_RETRIES,
self.can_attempt_resume_on_error,
is_online,
) {
RecoveryAction::RetryNow => {
log::warn!(
"MultiAgent request failed, retrying (attempt {}/{}) - Error: {e:?}",
self.retry_count + 1,
MAX_RETRIES
);
scope.set_tag("error", format!("{e:?}"));
scope.set_tag("is_retryable", e.is_retryable());
scope.set_tag("is_online", is_online);
scope.set_tag("retry_count", self.retry_count);
},
|| {
report_error!(anyhow!(e.clone()).context(format!(
"MultiAgent request failed after {} retries",
self.retry_count
)));
},
);
#[cfg(not(feature = "crash_reporting"))]
{
report_error!(anyhow!(e.clone()).context(format!(
"MultiAgent request failed after {} retries",
self.retry_count
)));
// Only emit error telemetry here if we're retrying.
// Final errors that aren't being retried are emitted elsewhere.
self.emit_retryable_agent_mode_error_telemetry(format!("{e:?}"), ctx);
self.retry(ctx);
// Don't emit the error event, we're retrying
return;
}
RecoveryAction::RetryWhenOnline => {
log::warn!(
"MultiAgent request failed while offline; retrying (attempt {}/{}) once connectivity returns - Error: {e:?}",
self.retry_count + 1,
MAX_RETRIES
);
self.emit_retryable_agent_mode_error_telemetry(format!("{e:?}"), ctx);
self.defer_retry_until_online(ctx);
return;
}
RecoveryAction::Resume => {
// Recoverable failure after client actions: we'll resume the
// conversation once the stream finishes rather than surface the
// error, so the UI suppresses the banner. Log it so the
// auto-recovery isn't completely silent.
log::warn!(
"MultiAgent request failed after client actions; resuming conversation after stream finishes - Error: {e:?}"
);
// The resume spawn itself waits for connectivity.
self.should_resume_conversation_after_stream_finished = true;
}
RecoveryAction::Fail => {}
}
self.error_event_emitted = true;
self.report_request_failure(e, is_online);
ctx.emit(ResponseStreamEvent::ReceivedEvent(Consumable::new(event)));
}
@@ -451,9 +474,135 @@ impl ResponseStream {
log::info!("[bedrock-debug] on_response_stream_complete: stale request_id, ignoring");
return;
}
// A retry is parked waiting for connectivity; the request is logically still
// active, so don't complete the stream for the failed attempt.
if self.deferred_retry_pending {
return;
}
// The server always sends a StreamFinished event before ending the response,
// but a transport cut between chunks surfaces as a clean EOF. Synthesize the
// failure and recover like any transient error.
if !self.stream_finished_received && !self.error_event_emitted {
log::warn!(
"generate_multi_agent_output stream ended without emitting StreamFinished event."
);
let unexpected_eof = Arc::new(AIApiError::UnexpectedEof);
let is_online = NetworkStatus::as_ref(ctx).is_online();
match recovery_action(
self.has_received_client_actions,
unexpected_eof.is_recoverable(),
self.retry_count < MAX_RETRIES,
self.can_attempt_resume_on_error,
is_online,
) {
RecoveryAction::RetryNow => {
log::warn!(
"MultiAgent request failed, retrying (attempt {}/{}) - Error: {unexpected_eof:?}",
self.retry_count + 1,
MAX_RETRIES
);
self.emit_retryable_agent_mode_error_telemetry(
format!("{unexpected_eof:?}"),
ctx,
);
self.retry(ctx);
return;
}
RecoveryAction::RetryWhenOnline => {
log::warn!(
"MultiAgent request failed while offline; retrying (attempt {}/{}) once connectivity returns - Error: {unexpected_eof:?}",
self.retry_count + 1,
MAX_RETRIES
);
self.emit_retryable_agent_mode_error_telemetry(
format!("{unexpected_eof:?}"),
ctx,
);
self.defer_retry_until_online(ctx);
return;
}
RecoveryAction::Resume => {
// Recoverable truncation after client actions: we'll resume the
// conversation once the stream finishes rather than surface the
// error, so the UI suppresses the banner. Log it so the
// auto-recovery isn't completely silent.
log::warn!(
"MultiAgent request truncated after client actions; resuming conversation after stream finishes - Error: {unexpected_eof:?}"
);
self.should_resume_conversation_after_stream_finished = true;
self.error_event_emitted = true;
self.report_request_failure(&unexpected_eof, is_online);
ctx.emit(ResponseStreamEvent::ReceivedEvent(Consumable::new(Err(
unexpected_eof,
))));
}
RecoveryAction::Fail => {
self.error_event_emitted = true;
self.report_request_failure(&unexpected_eof, is_online);
ctx.emit(ResponseStreamEvent::ReceivedEvent(Consumable::new(Err(
unexpected_eof,
))));
}
}
}
ctx.emit(ResponseStreamEvent::AfterStreamFinished { cancellation: None });
self.cancellation_tx = None;
}
/// Reports a non-retried request failure to crash reporting with classification
/// tags.
#[cfg_attr(not(feature = "crash_reporting"), expect(unused_variables))]
fn report_request_failure(&self, error: &Arc<AIApiError>, is_online: bool) {
#[cfg(feature = "crash_reporting")]
sentry::with_scope(
|scope| {
scope.set_tag(
"has_received_client_actions",
self.has_received_client_actions,
);
scope.set_tag("error", format!("{error:?}"));
scope.set_tag("is_recoverable", error.is_recoverable());
scope.set_tag(
"will_attempt_resume",
self.should_resume_conversation_after_stream_finished,
);
scope.set_tag("is_online", is_online);
scope.set_tag("retry_count", self.retry_count);
},
|| {
report_error!(anyhow!(error.clone()).context(format!(
"MultiAgent request failed after {} retries",
self.retry_count
)));
},
);
#[cfg(not(feature = "crash_reporting"))]
{
report_error!(anyhow!(error.clone()).context(format!(
"MultiAgent request failed after {} retries",
self.retry_count
)));
}
}
/// Parks a retry until connectivity returns; cancellation invalidates the parked
/// retry through `current_request_id`.
fn defer_retry_until_online(&mut self, ctx: &mut ModelContext<Self>) {
self.deferred_retry_pending = true;
ctx.emit(ResponseStreamEvent::WaitingForNetwork { waiting: true });
let request_id_at_defer = self.current_request_id;
let wait_for_online = NetworkStatus::as_ref(ctx).wait_until_online();
let _ = ctx.spawn(wait_for_online, move |me, _, ctx| {
// Cancelled or superseded while waiting — drop the parked retry.
if request_id_at_defer.is_none() || me.current_request_id != request_id_at_defer {
return;
}
ctx.emit(ResponseStreamEvent::WaitingForNetwork { waiting: false });
me.retry(ctx);
});
}
}
#[derive(Debug)]
@@ -492,6 +641,16 @@ pub struct StreamCancellation {
#[derive(Debug, Clone)]
pub enum ResponseStreamEvent {
ReceivedEvent(Consumable<api::Event>),
/// A retry is parked until connectivity returns (`waiting: true`) or has just
/// fired (`waiting: false`). The controller mirrors this on the conversation
/// status (`TransientError` ↔ `InProgress`).
///
/// Only emitted from `defer_retry_until_online`, i.e. always after a recoverable
/// request failure while offline — never speculatively before an attempt. Consumers
/// can therefore treat `waiting: true` as a transient-error (reconnecting) state.
WaitingForNetwork {
waiting: bool,
},
AfterStreamFinished {
/// Some for cancellation (with context), None for natural completion (uses dynamic lookup).
cancellation: Option<StreamCancellation>,
@@ -501,3 +660,7 @@ pub enum ResponseStreamEvent {
impl Entity for ResponseStream {
type Event = ResponseStreamEvent;
}
#[cfg(test)]
#[path = "response_stream_tests.rs"]
mod tests;