diff --git a/Cargo.lock b/Cargo.lock index 9de22130..af6adcd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1120,9 +1120,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "as-raw-xcb-connection" @@ -4024,7 +4024,7 @@ dependencies = [ "ouroboros", "proc-macro2", "quote", - "rkyv 0.8.16", + "rkyv 0.8.17", "strsim 0.10.0", "syn 2.0.118", "thiserror 1.0.69", @@ -4783,9 +4783,9 @@ dependencies = [ [[package]] name = "embed-resource" -version = "3.0.9" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a88c8d26de40ed18fe748c547845aa39de1db3afd958f8cb91579f3644bcb" +checksum = "fbfdaacccebec3b28e4866b8973543c7647797db5ada1bdab552e48fe665fbbd" dependencies = [ "cc", "memchr", @@ -6566,7 +6566,7 @@ dependencies = [ "resvg", "rstar", "rust-embed 8.11.0", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "schemars", "serde", "serde_json", @@ -7751,9 +7751,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -8119,9 +8119,9 @@ dependencies = [ [[package]] name = "inotify-sys" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fda9741ca16536952da2ecaa6105c2f4653fa6f0724681df6d2414c4106d0b0" +checksum = "9ea94e891b3606826e9c998be69ddca42247dad8ad50b1649a5cb7e1c9ae06fd" dependencies = [ "libc", ] @@ -8577,9 +8577,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" [[package]] name = "jiff-tzdb-platform" @@ -9827,9 +9827,9 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "naga" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd91265cc2454558f659b3b4b9640f0ddb8cc6521277f166b8a8c181c898079" +checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df" dependencies = [ "arrayvec", "bit-set 0.9.1", @@ -10163,9 +10163,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" dependencies = [ "num-integer", "num-traits", @@ -11587,7 +11587,7 @@ checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64 0.22.1", "indexmap 2.14.0", - "quick-xml 0.39.4", + "quick-xml", "serde", "time", ] @@ -12124,9 +12124,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.215" +version = "2.1.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caff7403e06671f170c65dc7bf475ed31d6e108c7e0d2440fb4df8ba56cfded6" +checksum = "9826c2fe4dade07e9da1af2e18427257877bc8bc27aa810f6fbb52d907a480cc" dependencies = [ "psl-types", ] @@ -12232,15 +12232,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.39.4" @@ -12261,7 +12252,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls", "socket2 0.6.4", "thiserror 2.0.18", @@ -12282,7 +12273,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls", "rustls-pki-types", "slab", @@ -12358,9 +12349,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rancor" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" dependencies = [ "ptr_meta 0.3.1", ] @@ -12867,9 +12858,9 @@ dependencies = [ [[package]] name = "rend" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +checksum = "663ba70707f96e871406fe10d68128412e619b06d1d47cb91c3a4c6501176240" dependencies = [ "bytecheck 0.8.2", ] @@ -13071,9 +13062,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck 0.8.2", "bytes", @@ -13082,8 +13073,8 @@ dependencies = [ "munge", "ptr_meta 0.3.1", "rancor", - "rend 0.5.3", - "rkyv_derive 0.8.16", + "rend 0.5.4", + "rkyv_derive 0.8.17", "tinyvec", "uuid", ] @@ -13101,9 +13092,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", @@ -13338,9 +13329,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -14958,7 +14949,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "sketches-ddsketch", @@ -15092,11 +15083,10 @@ dependencies = [ [[package]] name = "tauri-winrt-notification" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1e66e07de489fe43a46678dd0b8df65e0c973909df1b60ba33874e297ba9b9" +checksum = "9ed071c670382e85fc2f48ae706492d8c338f4f89bf72520d32f8abfe880aade" dependencies = [ - "quick-xml 0.37.5", "thiserror 2.0.18", "windows 0.61.3", "windows-version", @@ -17088,7 +17078,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" dependencies = [ "proc-macro2", - "quick-xml 0.39.4", + "quick-xml", "quote", ] @@ -17284,9 +17274,9 @@ dependencies = [ [[package]] name = "wgpu" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb3feacc458f7bee8bc1737149b42b6c731aa461039a4264a67bb6681646b250" +checksum = "76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07" dependencies = [ "arrayvec", "bitflags 2.13.0", @@ -17313,9 +17303,9 @@ dependencies = [ [[package]] name = "wgpu-core" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02da3ad1b568337f25513b317870960ef87073ea0945502e44b864b67a8c77b7" +checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7" dependencies = [ "arrayvec", "bit-set 0.9.1", @@ -17347,45 +17337,45 @@ dependencies = [ [[package]] name = "wgpu-core-deps-apple" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e51b5447e144b3dbba4feb01f80f4fa21696fa0cd99afb2c3df1affd6fdb28" +checksum = "f5e39e26c4c0e07589e67d18546cf79ff45383659fc72fca4dd293358a0347f3" dependencies = [ "wgpu-hal", ] [[package]] name = "wgpu-core-deps-emscripten" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3487cd6293a963bc5c0c0396f6a2192043c50003c07f4efdccbad3d90ec9d819" +checksum = "01e09be551dc939498bdd5f6b2c66e55ab275dad25825267a08605a80fc9f0af" dependencies = [ "wgpu-hal", ] [[package]] name = "wgpu-core-deps-wasm" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2f2fb042f36920771deb0b966543c5751b18f3d327760ffc90f74e20b2dcd4" +checksum = "af1fb1798be2a912497d4c224f72d39bb0cb34af50e8bcc29865bc339c943059" dependencies = [ "wgpu-hal", ] [[package]] name = "wgpu-core-deps-windows-linux-android" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb01076d0aa08b0ba9bd741e178b5cc440f5abe99d9581323a4c8b5d1a1916" +checksum = "4e592c1bbef6ad047647ae6e666ebd8cee7a32bb4544d9700ec96cbf73230257" dependencies = [ "wgpu-hal", ] [[package]] name = "wgpu-hal" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8e1a9e7a8512f276f7c62e018c7fa8d60954303fed2e5750114332049193f" +checksum = "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d" dependencies = [ "android_system_properties", "arrayvec", @@ -17437,9 +17427,9 @@ dependencies = [ [[package]] name = "wgpu-naga-bridge" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59c654c483f058800972c3645e95388a7eca31bf9fe1933bc20e036588a0be02" +checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161" dependencies = [ "naga", "wgpu-types", @@ -17447,9 +17437,9 @@ dependencies = [ [[package]] name = "wgpu-types" -version = "29.0.3" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9bcc31518a0e9735aefebedb5f7a9ef3ed1c42549c9f4c882fa9060ceaac639" +checksum = "84bf84cd9ca8ca45e2b223a3868f1adf9bfc0c66aeac212e76ee7e40fdadf8f5" dependencies = [ "bitflags 2.13.0", "bytemuck", diff --git a/app/src/ai/bedrock/client.rs b/app/src/ai/bedrock/client.rs index 90173754..6cf06950 100644 --- a/app/src/ai/bedrock/client.rs +++ b/app/src/ai/bedrock/client.rs @@ -268,6 +268,15 @@ impl BedrockClient { ); } } + + super::crash_log::log_crash( + "BedrockApiError", + &msg, + &effective_model_id, + messages.len(), + None, + ); + if msg.contains("AccessDenied") || msg.contains("access denied") { BedrockError::AccessDenied(msg) } else if msg.contains("ThrottlingException") || msg.contains("throttl") { diff --git a/app/src/ai/bedrock/crash_log.rs b/app/src/ai/bedrock/crash_log.rs new file mode 100644 index 00000000..2994959f --- /dev/null +++ b/app/src/ai/bedrock/crash_log.rs @@ -0,0 +1,95 @@ +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::PathBuf; + +use chrono::Local; + +const CRASH_LOG_DIR: &str = "crash-logs"; +const MAX_CRASH_LOGS: usize = 20; + +fn crash_log_dir() -> Option { + galaxy_core::paths::galaxy_home_config_dir().map(|dir| dir.join(CRASH_LOG_DIR)) +} + +pub fn log_crash( + error_type: &str, + error_message: &str, + model_id: &str, + message_count: usize, + context_tokens: Option, +) { + let Some(dir) = crash_log_dir() else { + log::warn!("[crash-log] Could not determine crash log directory"); + return; + }; + + if let Err(e) = fs::create_dir_all(&dir) { + log::warn!("[crash-log] Failed to create crash log directory: {e}"); + return; + } + + rotate_logs(&dir); + + let timestamp = Local::now(); + let filename = format!("crash_{}.log", timestamp.format("%Y%m%d_%H%M%S")); + let path = dir.join(&filename); + + let content = format!( + "=== Galaxy Crash Log ===\n\ + Timestamp: {}\n\ + Error Type: {}\n\ + Model: {}\n\ + Message Count: {}\n\ + Context Tokens: {}\n\ + \n\ + Error Details:\n\ + {}\n\ + ========================\n", + timestamp.format("%Y-%m-%d %H:%M:%S %Z"), + error_type, + model_id, + message_count, + context_tokens + .map(|t| t.to_string()) + .unwrap_or_else(|| "unknown".to_string()), + error_message, + ); + + match OpenOptions::new().create(true).write(true).open(&path) { + Ok(mut file) => { + if let Err(e) = file.write_all(content.as_bytes()) { + log::warn!("[crash-log] Failed to write crash log: {e}"); + } else { + log::info!("[crash-log] Wrote crash log to {}", path.display()); + } + } + Err(e) => { + log::warn!("[crash-log] Failed to open crash log file: {e}"); + } + } +} + +fn rotate_logs(dir: &std::path::Path) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + + let mut logs: Vec = entries + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("crash_") && n.ends_with(".log")) + }) + .collect(); + + logs.sort(); + + while logs.len() >= MAX_CRASH_LOGS { + if let Some(oldest) = logs.first() { + let _ = fs::remove_file(oldest); + logs.remove(0); + } + } +} diff --git a/app/src/ai/bedrock/mod.rs b/app/src/ai/bedrock/mod.rs index 1745bcef..38c975a6 100644 --- a/app/src/ai/bedrock/mod.rs +++ b/app/src/ai/bedrock/mod.rs @@ -1,5 +1,6 @@ pub mod client; pub mod convert; +pub mod crash_log; pub mod diagnostic; pub mod external_config; pub mod models; diff --git a/app/src/ai/bedrock/request_translator.rs b/app/src/ai/bedrock/request_translator.rs index 1db00522..7b17adbd 100644 --- a/app/src/ai/bedrock/request_translator.rs +++ b/app/src/ai/bedrock/request_translator.rs @@ -1562,6 +1562,41 @@ fn extract_tool_result_content(result: &api::request::input::ToolCallResult) -> None => ("Agent completed.".to_string(), false), } } + api::request::input::tool_call_result::Result::AskUserQuestion(ask_result) => { + match &ask_result.result { + Some(api::ask_user_question_result::Result::Success(success)) => { + let answers_text: Vec = success + .answers + .iter() + .map(|item| { + let answer_str = match &item.answer { + Some(api::ask_user_question_result::answer_item::Answer::MultipleChoice(mc)) => { + let mut parts = mc.selected_options.clone(); + if !mc.other_text.is_empty() { + parts.push(mc.other_text.clone()); + } + parts.join(", ") + } + Some(api::ask_user_question_result::answer_item::Answer::Skipped(_)) => { + "Skipped".to_string() + } + None => "No answer provided".to_string(), + }; + if item.question_id.is_empty() { + answer_str + } else { + format!("{}: {}", item.question_id, answer_str) + } + }) + .collect(); + (format!("User's answers:\n{}", answers_text.join("\n")), false) + } + Some(api::ask_user_question_result::Result::Error(error)) => { + (format!("User question error: {}", error.message), true) + } + None => ("User did not answer.".to_string(), true), + } + } _ => ("Tool completed successfully.".to_string(), false), } } else { diff --git a/app/src/ai/bedrock/response_translator.rs b/app/src/ai/bedrock/response_translator.rs index 46f8103e..0bbe47c5 100644 --- a/app/src/ai/bedrock/response_translator.rs +++ b/app/src/ai/bedrock/response_translator.rs @@ -397,6 +397,13 @@ pub fn bedrock_stream_to_response_events( ); } } + super::crash_log::log_crash( + "BedrockStreamError", + &format!("{e}"), + &model_id, + event_count as usize, + Some(input_tokens as u32), + ); if !buffered_text.is_empty() { let msg_id = current_text_message_id .clone() diff --git a/app/src/ai/blocklist/controller.rs b/app/src/ai/blocklist/controller.rs index 016dfd95..7bc4293e 100644 --- a/app/src/ai/blocklist/controller.rs +++ b/app/src/ai/blocklist/controller.rs @@ -404,6 +404,8 @@ pub struct BlocklistAIController { /// Per-conversation loop detection state for preventing recursive tool failures. loop_detection: HashMap, + /// Per-conversation error retry count for injecting corrective messages on failure. + error_retry_counts: HashMap, /// Passive suggestion results that should be included with the next request /// for a given conversation (e.g. accepted/iterated code diffs that weren't /// auto-resumed). @@ -684,6 +686,7 @@ impl BlocklistAIController { pending_passive_follow_ups: HashSet::new(), pending_passive_suggestion_results: HashMap::new(), loop_detection: HashMap::new(), + error_retry_counts: HashMap::new(), } } @@ -1233,8 +1236,9 @@ impl BlocklistAIController { queued_query_id: Option, ctx: &mut ModelContext, ) { - // User sending a new query resets loop detection — fresh context. + // User sending a new query resets loop detection and error retry state — fresh context. self.loop_detection.remove(&conversation_id); + self.error_retry_counts.remove(&conversation_id); let is_viewer = self .terminal_model @@ -3132,42 +3136,145 @@ impl BlocklistAIController { }); } - // A resume scheduled for this failure keeps the conversation in - // the non-terminal TransientError status instead of Error. - let recovery_pending = response_stream - .as_ref(ctx) - .should_resume_conversation_after_stream_finished(); - let mut renderable_error: RenderableAIError = (&e).into(); - if let RenderableAIError::Other { - will_attempt_resume, - waiting_for_network, - .. - } - | RenderableAIError::TransientNetworkError { - will_attempt_resume, - waiting_for_network, - .. - } = &mut renderable_error - { - // Rendering-only hints; state machine consumers key off the - // TransientError conversation status instead. - *will_attempt_resume |= recovery_pending; - if recovery_pending { - let network_status = NetworkStatus::as_ref(ctx); - *waiting_for_network = !network_status.is_online(); - } - } + // Check if this error is eligible for corrective retry. + // Similar to loop detection, inject a message telling the LLM + // to try a different approach rather than just failing. + let error_str = format!("{e}"); + let is_corrective_retry_candidate = !matches!( + e.as_ref(), + AIApiError::QuotaLimit { .. } + ) && (error_str.contains("ValidationException") + || error_str.contains("validation") + || error_str.contains("context window") + || error_str.contains("too many tokens") + || error_str.contains("input is too long") + || error_str.contains("throttl") + || error_str.contains("ThrottlingException")); - history_model.update(ctx, |history_model, ctx| { - history_model.mark_response_stream_completed_with_error( - renderable_error, - recovery_pending, - &stream_id, + const MAX_ERROR_RETRIES: usize = 2; + let retry_count = self + .error_retry_counts + .entry(conversation_id) + .or_insert(0); + let should_corrective_retry = + is_corrective_retry_candidate && *retry_count < MAX_ERROR_RETRIES; + + if should_corrective_retry { + *retry_count += 1; + let retry_num = *retry_count; + log::warn!( + "[error-retry] Attempting corrective retry {}/{} for conversation {:?}: {}", + retry_num, + MAX_ERROR_RETRIES, conversation_id, - self.terminal_surface_id, - ctx, + error_str ); - }); + + // Mark the error on the conversation but with recovery pending + let renderable_error = RenderableAIError::Other { + error_message: format!( + "Error encountered, retrying with different approach (attempt {}/{})", + retry_num, MAX_ERROR_RETRIES + ), + will_attempt_resume: true, + waiting_for_network: false, + is_user_error: false, + }; + history_model.update(ctx, |history_model, ctx| { + history_model.mark_response_stream_completed_with_error( + renderable_error, + /*recovery_pending*/ true, + &stream_id, + conversation_id, + self.terminal_surface_id, + ctx, + ); + }); + + // Inject a corrective message and resume + if let Some(conversation) = + BlocklistAIHistoryModel::as_ref(ctx).conversation(&conversation_id) + { + let root_task_id = conversation.get_root_task_id().clone(); + let corrective_msg = format!( + "[SYSTEM] The previous request resulted in an error: {}\n\n\ + Please try a completely different approach to accomplish the goal. \ + If the error is related to context size, reduce the amount of content \ + you are working with (read fewer files, use smaller commands, break \ + the task into smaller steps). If you cannot find an alternative, \ + explain to the user what is failing and why.", + error_str + ); + + let inputs = vec![ + AIAgentInput::UserQuery { + query: corrective_msg, + context: Arc::from([]), + static_query_type: None, + referenced_attachments: HashMap::new(), + user_query_mode: UserQueryMode::Normal, + running_command: None, + intended_agent: None, + }, + ]; + + let _ = self.send_request_input( + RequestInput::for_task( + inputs, + root_task_id, + &self.active_session, + self.get_current_response_initiator(), + conversation_id, + self.terminal_surface_id, + ctx, + ), + None, + /*can_attempt_resume_on_error*/ false, + /*is_queued_prompt*/ false, + ctx, + ); + } + } else { + // Clear retry count on non-retryable errors or exhausted retries + self.error_retry_counts.remove(&conversation_id); + + // A resume scheduled for this failure keeps the conversation in + // the non-terminal TransientError status instead of Error. + let recovery_pending = response_stream + .as_ref(ctx) + .should_resume_conversation_after_stream_finished(); + let mut renderable_error: RenderableAIError = (&e).into(); + if let RenderableAIError::Other { + will_attempt_resume, + waiting_for_network, + .. + } + | RenderableAIError::TransientNetworkError { + will_attempt_resume, + waiting_for_network, + .. + } = &mut renderable_error + { + // Rendering-only hints; state machine consumers key off the + // TransientError conversation status instead. + *will_attempt_resume |= recovery_pending; + if recovery_pending { + let network_status = NetworkStatus::as_ref(ctx); + *waiting_for_network = !network_status.is_online(); + } + } + + history_model.update(ctx, |history_model, ctx| { + history_model.mark_response_stream_completed_with_error( + renderable_error, + recovery_pending, + &stream_id, + conversation_id, + self.terminal_surface_id, + ctx, + ); + }); + } } } } @@ -3510,6 +3617,13 @@ impl BlocklistAIController { } Some(warp_multi_agent_api::response_event::stream_finished::Reason::ContextWindowExceeded(_)) => { let error_message = "Input exceeded context window limit."; + crate::ai::bedrock::crash_log::log_crash( + "ContextWindowExceeded", + error_message, + "unknown", + 0, + None, + ); history_model.update(ctx, |history_model, ctx| { history_model.mark_response_stream_completed_with_error( RenderableAIError::ContextWindowExceeded(error_message.to_owned()), @@ -3596,6 +3710,13 @@ impl BlocklistAIController { let error_message = format!( "Response stream finished unexpectedly with internal error: {message}", ); + crate::ai::bedrock::crash_log::log_crash( + "InternalError", + &error_message, + "unknown", + 0, + None, + ); history_model.update(ctx, |history_model, ctx| { history_model.mark_response_stream_completed_with_error( RenderableAIError::Other { @@ -3614,6 +3735,13 @@ impl BlocklistAIController { } Some(warp_multi_agent_api::response_event::stream_finished::Reason::MaxTokenLimit(_)) => { let error_message = "Input exceeded context window limit."; + crate::ai::bedrock::crash_log::log_crash( + "MaxTokenLimit", + error_message, + "unknown", + 0, + None, + ); history_model.update(ctx, |history_model, ctx| { history_model.mark_response_stream_completed_with_error( RenderableAIError::ContextWindowExceeded(error_message.to_owned()), diff --git a/app/src/ai/blocklist/controller/response_stream.rs b/app/src/ai/blocklist/controller/response_stream.rs index 3f8dc820..b40af2f7 100644 --- a/app/src/ai/blocklist/controller/response_stream.rs +++ b/app/src/ai/blocklist/controller/response_stream.rs @@ -16,6 +16,7 @@ use crate::ai::agent::api::{self, generate_multi_agent_output, ConvertToAPITypeE use crate::ai::agent::conversation::AIConversationId; use crate::ai::agent::{AIIdentifiers, CancellationReason}; use crate::ai::bedrock::client::BedrockClientConfig; +use crate::ai::llms::LLMPreferences; use crate::ai::openai::client::OpenAIClientConfig; use crate::ai::provider::ProviderConfig; use crate::network::NetworkStatus; @@ -174,32 +175,18 @@ impl ResponseStream { fn resolve_provider_config(model_id: &str, ctx: &ModelContext) -> ProviderConfig { let settings = AISettings::as_ref(ctx); - // Check if OpenAI/LiteLLM provider is enabled + // Check if this specific model has an OpenAI-compatible routing entry. + // This allows OpenAI/LiteLLM models to coexist with Bedrock models — + // only models fetched from the OpenAI endpoint route through it. if *settings.openai_enabled.value() { - let base_url = settings.openai_base_url.value().clone(); - let api_key = { - let key = settings.openai_api_key.value().clone(); - if key.is_empty() { - None - } else { - Some(key) - } - }; - // Use the model override from settings if set, otherwise use the selected model ID. - // This allows LiteLLM models to pass through their actual model_id to the proxy. - let model = { - let m = settings.openai_model.value().clone(); - if m.is_empty() { - Some(model_id.to_string()) - } else { - Some(m) - } - }; - return ProviderConfig::OpenAI(OpenAIClientConfig { - base_url, - api_key, - model, - }); + let llm_prefs = LLMPreferences::as_ref(ctx); + if let Some(client_config) = llm_prefs.openai_client_config_for_model(model_id) { + return ProviderConfig::OpenAI(OpenAIClientConfig { + base_url: client_config.base_url.clone(), + api_key: client_config.api_key.clone(), + model: Some(model_id.to_string()), + }); + } } // Fall back to Bedrock diff --git a/app/src/ai/llms.rs b/app/src/ai/llms.rs index 308f684c..b9140c01 100644 --- a/app/src/ai/llms.rs +++ b/app/src/ai/llms.rs @@ -21,7 +21,7 @@ use crate::auth::auth_manager::{AuthManager, AuthManagerEvent}; use crate::auth::AuthStateProvider; use crate::network::{NetworkStatus, NetworkStatusEvent, NetworkStatusKind}; use crate::server::server_api::ServerApiProvider; -use crate::settings::{BedrockModelConfig, OpenAIModelConfig, OpenAIProviderConfig}; +use crate::settings::{BedrockModelConfig, OpenAIModelConfig}; use crate::user_config::{WarpConfig, WarpConfigUpdateEvent}; use crate::workspaces::user_workspaces::{UserWorkspaces, UserWorkspacesEvent}; use crate::{report_error, AISettings}; @@ -601,6 +601,10 @@ pub struct LLMPreferences { custom_model_routers: Vec, #[cfg(not(target_family = "wasm"))] openai_provider_routing: HashMap, + /// Models fetched from the OpenAI-compatible /models endpoint at runtime. + /// Stored in memory only — not persisted to TOML. + #[cfg(not(target_family = "wasm"))] + fetched_openai_models: Vec, } impl LLMPreferences { @@ -657,6 +661,28 @@ impl LLMPreferences { }); } + // Re-inject provider models when Bedrock or OpenAI enabled state changes. + #[cfg(not(target_family = "wasm"))] + ctx.subscribe_to_model(&AISettings::handle(ctx), |me, _, event, ctx| { + use crate::settings::AISettingsChangedEvent; + if matches!( + event, + AISettingsChangedEvent::BedrockEnabled { .. } + | AISettingsChangedEvent::OpenAIEnabled { .. } + | AISettingsChangedEvent::OpenAIBaseUrl { .. } + ) { + me.inject_bedrock_models(ctx); + me.inject_openai_models(ctx); + if matches!(event, AISettingsChangedEvent::OpenAIEnabled { .. } | AISettingsChangedEvent::OpenAIBaseUrl { .. }) { + me.fetch_openai_models_from_endpoint(ctx); + } + // Safety: ensure the default model is still present in choices. + // If all provider models were removed, the default_id would dangle. + me.ensure_default_model_present(); + ctx.emit(LLMPreferencesEvent::UpdatedAvailableLLMs); + } + }); + let base_llm_for_terminal_view = HashMap::new(); let custom_llms = build_custom_llm_infos(ApiKeyManager::as_ref(ctx).keys()); @@ -668,6 +694,8 @@ impl LLMPreferences { custom_model_routers: Vec::new(), #[cfg(not(target_family = "wasm"))] openai_provider_routing: HashMap::new(), + #[cfg(not(target_family = "wasm"))] + fetched_openai_models: Vec::new(), }; // Seed from any already-loaded local config (the async load emits @@ -688,6 +716,7 @@ impl LLMPreferences { Self::ensure_default_models_in_settings(ctx); me.inject_bedrock_models(ctx); me.inject_openai_models(ctx); + me.fetch_openai_models_from_endpoint(ctx); } me @@ -932,27 +961,11 @@ impl LLMPreferences { return; } - // Collect all (provider_name, base_url, api_key, models) tuples from both config paths. + // Models come exclusively from the in-memory /models endpoint fetch. let mut provider_entries: Vec<(String, String, Option, Vec)> = Vec::new(); - // Path 1: Multi-provider `ai.providers[]` - let providers: Vec = settings.openai_providers.value().clone(); - for provider in providers { - if provider.models.is_empty() { - continue; - } - provider_entries.push(( - provider.name, - provider.base_url, - provider.api_key, - provider.models, - )); - } - - // Path 2: Legacy single-provider `ai.openai.{base_url, models}` - let legacy_models: Vec = settings.openai_models.value().clone(); - if !legacy_models.is_empty() { + if !self.fetched_openai_models.is_empty() { let base_url = settings.openai_base_url.value().clone(); let api_key = { let key = settings.openai_api_key.value().clone(); @@ -967,7 +980,7 @@ impl LLMPreferences { } else { "LiteLLM".to_string() }; - provider_entries.push((name, base_url, api_key, legacy_models)); + provider_entries.push((name, base_url, api_key, self.fetched_openai_models.clone())); } if provider_entries.is_empty() { @@ -975,6 +988,7 @@ impl LLMPreferences { } let mut total_injected = 0; + let mut seen_model_ids: HashSet = HashSet::new(); for (provider_name, base_url, api_key, models) in provider_entries { let client_config = OpenAIClientConfig { base_url: base_url.clone(), @@ -983,6 +997,10 @@ impl LLMPreferences { }; for model in &models { + if !seen_model_ids.insert(model.model_id.clone()) { + continue; + } + // Register the routing entry self.openai_provider_routing .insert(model.model_id.clone(), client_config.clone()); @@ -1026,6 +1044,36 @@ impl LLMPreferences { log::info!("[openai/litellm] Injected {total_injected} model(s) into available choices"); } + /// Ensures the default model ID in each feature's choices still points to + /// an existing entry. If the default was removed (e.g. provider disabled), + /// switch to the first remaining choice. + #[cfg(not(target_family = "wasm"))] + fn ensure_default_model_present(&mut self) { + fn fix_default(feature: &mut AvailableLLMs) { + if feature.choices.is_empty() { + return; + } + let default_exists = feature + .choices + .iter() + .any(|m| m.id == feature.default_id); + if !default_exists { + let new_default = feature.choices[0].id.clone(); + log::info!( + "[llm] Default model {:?} no longer available, switching to {:?}", + feature.default_id, + new_default + ); + feature.default_id = new_default; + } + } + fix_default(&mut self.models_by_feature.agent_mode); + fix_default(&mut self.models_by_feature.coding); + if let Some(ref mut cli) = self.models_by_feature.cli_agent { + fix_default(cli); + } + } + /// Returns the OpenAI client config for a given model ID, if it was injected /// from an OpenAI-compatible provider. #[cfg(not(target_family = "wasm"))] @@ -1036,6 +1084,130 @@ impl LLMPreferences { self.openai_provider_routing.get(model_id) } + /// Fetches available models from the configured OpenAI-compatible /models endpoint + /// and stores them in memory. Called at startup and when the user clicks "Fetch Models". + #[cfg(not(target_family = "wasm"))] + pub fn fetch_openai_models_from_endpoint(&mut self, ctx: &mut ModelContext) { + let settings = AISettings::as_ref(ctx); + if !*settings.openai_enabled.value() { + return; + } + + let base_url = settings.openai_base_url.value().clone(); + if base_url.is_empty() { + return; + } + + let api_key = { + let key = settings.openai_api_key.value().clone(); + if key.is_empty() { + None + } else { + Some(key) + } + }; + + let _ = ctx.spawn( + async move { + let url = format!("{}/models", base_url.trim_end_matches('/')); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .unwrap_or_default(); + let mut request = client.get(&url); + if let Some(ref key) = api_key { + request = request.header("Authorization", format!("Bearer {key}")); + } + + let response = match request.send().await { + Ok(r) => r, + Err(e) => { + log::warn!("[openai/litellm] Failed to fetch models from endpoint: {e}"); + return Vec::new(); + } + }; + + if !response.status().is_success() { + log::warn!( + "[openai/litellm] Model fetch returned HTTP {}", + response.status() + ); + return Vec::new(); + } + + let body: serde_json::Value = match response.json().await { + Ok(v) => v, + Err(e) => { + log::warn!("[openai/litellm] Failed to parse models response: {e}"); + return Vec::new(); + } + }; + + let models: Vec = body["data"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .filter_map(|m| { + let id = m["id"].as_str()?; + let context_size = m["max_model_len"] + .as_u64() + .or_else(|| m["context_window"].as_u64()) + .or_else(|| m["max_input_tokens"].as_u64()) + .unwrap_or(200_000) as u32; + + let display_name = id + .split('/') + .next_back() + .unwrap_or(id) + .replace(['-', '_'], " "); + let display_name = display_name + .split_whitespace() + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().to_string() + chars.as_str(), + } + }) + .collect::>() + .join(" "); + + let provider = if id.contains("claude") || id.contains("anthropic") { + Some("anthropic".to_string()) + } else if id.contains("gpt") || id.contains("o1") || id.contains("o3") { + Some("openai".to_string()) + } else if id.contains("gemini") { + Some("google".to_string()) + } else { + None + }; + + Some(OpenAIModelConfig { + model_id: id.to_string(), + display_name, + vision_supported: m["supports_vision"].as_bool().unwrap_or(false), + context_size, + provider, + }) + }) + .collect(); + + log::info!( + "[openai/litellm] Fetched {} model(s) from endpoint", + models.len() + ); + models + }, + |me, models, ctx| { + if !models.is_empty() { + me.fetched_openai_models = models; + me.inject_openai_models(ctx); + ctx.emit(LLMPreferencesEvent::UpdatedAvailableLLMs); + } + }, + ); + } + /// Returns the `LLMInfo` for the base LLM to be used for an Agent Mode request. pub fn get_active_base_model<'a>( &'a self, diff --git a/app/src/ai/openai/request_translator.rs b/app/src/ai/openai/request_translator.rs index 87309383..759b0f01 100644 --- a/app/src/ai/openai/request_translator.rs +++ b/app/src/ai/openai/request_translator.rs @@ -49,41 +49,125 @@ fn remove_orphaned_tool_results(messages: &mut Vec) { } /// For any assistant tool_use that doesn't have a matching tool_result in a -/// subsequent user message, synthesize an error result. +/// subsequent user message, synthesize a result immediately after the tool_use. +/// This satisfies Bedrock's requirement (via LiteLLM) that tool_result blocks +/// appear immediately after the corresponding tool_use message. fn synthesize_missing_tool_results(messages: &mut Vec) { - let mut pending_tool_use_ids: Vec<(String, usize)> = Vec::new(); let mut answered_ids: std::collections::HashSet = std::collections::HashSet::new(); - // Collect all tool_use IDs and all answered IDs - for (i, msg) in messages.iter().enumerate() { - match msg.role { - MessageRole::Assistant => { - collect_tool_use_ids_with_index(&msg.content, i, &mut pending_tool_use_ids); - } - MessageRole::User => { - collect_tool_result_ids(&msg.content, &mut answered_ids); - } + // First pass: collect all existing tool_result IDs + for msg in messages.iter() { + if msg.role == MessageRole::User { + collect_tool_result_ids(&msg.content, &mut answered_ids); } } - // Find unanswered tool_uses and synthesize results - let mut synthetic_results: Vec = Vec::new(); - for (tool_use_id, _) in pending_tool_use_ids { - if !answered_ids.contains(&tool_use_id) { - synthetic_results.push(ConversationMessage { - role: MessageRole::User, - content: MessageContent::ToolResult { - tool_use_id, - content: "Tool call result unavailable (conversation was interrupted)." - .to_string(), - is_error: true, + // Second pass: walk through messages and insert synthetic results after + // assistant tool_use messages that have unanswered IDs. + let mut i = 0; + while i < messages.len() { + if messages[i].role != MessageRole::Assistant { + i += 1; + continue; + } + + let mut unanswered: Vec = Vec::new(); + collect_tool_use_ids_vec(&messages[i].content, &mut unanswered); + unanswered.retain(|id| !answered_ids.contains(id)); + + if unanswered.is_empty() { + i += 1; + continue; + } + + log::warn!( + "[openai] Synthesizing {} missing tool_result(s) after message {} for IDs: {:?}", + unanswered.len(), + i, + unanswered + ); + + let synthetic_parts: Vec = unanswered + .iter() + .map(|id| ContentPart::ToolResult { + tool_use_id: id.clone(), + content: "Tool call result unavailable (conversation was interrupted).".to_string(), + is_error: true, + }) + .collect(); + + let insert_idx = i + 1; + + // If next message is a user message, merge synthetic results into it + if insert_idx < messages.len() && messages[insert_idx].role == MessageRole::User { + match &mut messages[insert_idx].content { + MessageContent::MultiPart(parts) => { + let existing = std::mem::take(parts); + parts.extend(synthetic_parts); + parts.extend(existing); + } + existing => { + let existing_part = + match std::mem::replace(existing, MessageContent::Text(String::new())) { + MessageContent::Text(t) => ContentPart::Text(t), + MessageContent::ToolResult { + tool_use_id, + content, + is_error, + } => ContentPart::ToolResult { + tool_use_id, + content, + is_error, + }, + MessageContent::ToolUse { + tool_use_id, + name, + input, + } => ContentPart::ToolUse { + tool_use_id, + name, + input, + }, + MessageContent::MultiPart(_) => unreachable!(), + }; + let mut parts = synthetic_parts; + parts.push(existing_part); + *existing = MessageContent::MultiPart(parts); + } + } + } else { + // No user message follows — insert a new one + let content = if synthetic_parts.len() == 1 { + match synthetic_parts.into_iter().next().unwrap() { + ContentPart::ToolResult { + tool_use_id, + content, + is_error, + } => MessageContent::ToolResult { + tool_use_id, + content, + is_error, + }, + _ => unreachable!(), + } + } else { + MessageContent::MultiPart(synthetic_parts) + }; + messages.insert( + insert_idx, + ConversationMessage { + role: MessageRole::User, + content, }, - }); + ); } - } - if !synthetic_results.is_empty() { - messages.extend(synthetic_results); + // Mark these as answered so we don't double-synthesize + for id in unanswered { + answered_ids.insert(id); + } + + i += 2; // Skip past the inserted/modified message } } @@ -103,19 +187,15 @@ fn collect_tool_use_ids(content: &MessageContent, ids: &mut std::collections::Ha } } -fn collect_tool_use_ids_with_index( - content: &MessageContent, - index: usize, - ids: &mut Vec<(String, usize)>, -) { +fn collect_tool_use_ids_vec(content: &MessageContent, ids: &mut Vec) { match content { MessageContent::ToolUse { tool_use_id, .. } => { - ids.push((tool_use_id.clone(), index)); + ids.push(tool_use_id.clone()); } MessageContent::MultiPart(parts) => { for part in parts { if let ContentPart::ToolUse { tool_use_id, .. } = part { - ids.push((tool_use_id.clone(), index)); + ids.push(tool_use_id.clone()); } } } diff --git a/app/src/settings_view/ai_page.rs b/app/src/settings_view/ai_page.rs index cc2abdd0..5ec492f6 100644 --- a/app/src/settings_view/ai_page.rs +++ b/app/src/settings_view/ai_page.rs @@ -2732,118 +2732,13 @@ impl AISettingsPageView { } } - /// Fetches models from the LiteLLM endpoint and updates settings. + /// Fetches models from the LiteLLM endpoint and stores them in memory via LLMPreferences. fn fetch_litellm_models(&mut self, ctx: &mut ViewContext) { - let settings = AISettings::as_ref(ctx); - let base_url = settings.openai_base_url.value().clone(); - let api_key = { - let key = settings.openai_api_key.value().clone(); - if key.is_empty() { - None - } else { - Some(key) - } - }; + use crate::ai::llms::LLMPreferences; - let _ = ctx.spawn( - async move { - use crate::settings::ai::OpenAIModelConfig; - - let url = format!("{}/models", base_url.trim_end_matches('/')); - let client = reqwest::Client::new(); - let mut request = client.get(&url); - if let Some(ref key) = api_key { - request = request.header("Authorization", format!("Bearer {key}")); - } - - let response = match request.send().await { - Ok(r) => r, - Err(e) => { - log::error!("[litellm] Failed to fetch models: {e}"); - return Vec::new(); - } - }; - - if !response.status().is_success() { - log::error!("[litellm] Model fetch returned HTTP {}", response.status()); - return Vec::new(); - } - - let body: serde_json::Value = match response.json().await { - Ok(v) => v, - Err(e) => { - log::error!("[litellm] Failed to parse models response: {e}"); - return Vec::new(); - } - }; - - // LiteLLM /models endpoint returns OpenAI-compatible format: - // { "data": [{ "id": "model-name", "max_model_len": N, ... }] } - let models: Vec = body["data"] - .as_array() - .unwrap_or(&vec![]) - .iter() - .filter_map(|m| { - let id = m["id"].as_str()?; - // Try multiple context window fields used by different proxies - let context_size = m["max_model_len"] - .as_u64() - .or_else(|| m["context_window"].as_u64()) - .or_else(|| m["max_input_tokens"].as_u64()) - .unwrap_or(200_000) as u32; - - // Derive display name from model ID - let display_name = id - .split('/') - .next_back() - .unwrap_or(id) - .replace(['-', '_'], " "); - // Capitalize first letter of each word - let display_name = display_name - .split_whitespace() - .map(|word| { - let mut chars = word.chars(); - match chars.next() { - None => String::new(), - Some(c) => c.to_uppercase().to_string() + chars.as_str(), - } - }) - .collect::>() - .join(" "); - - // Infer provider from model ID prefix - let provider = if id.contains("claude") || id.contains("anthropic") { - Some("anthropic".to_string()) - } else if id.contains("gpt") || id.contains("o1") || id.contains("o3") { - Some("openai".to_string()) - } else if id.contains("gemini") { - Some("google".to_string()) - } else { - None - }; - - Some(OpenAIModelConfig { - model_id: id.to_string(), - display_name, - vision_supported: m["supports_vision"].as_bool().unwrap_or(false), - context_size, - provider, - }) - }) - .collect(); - - log::info!("[litellm] Fetched {} model(s) from {}", models.len(), url); - models - }, - |_view, models, ctx| { - if !models.is_empty() { - AISettings::handle(ctx).update(ctx, |settings, ctx| { - let _ = settings.openai_models.set_value(models, ctx); - }); - } - ctx.notify(); - }, - ); + LLMPreferences::handle(ctx).update(ctx, |llm_prefs, ctx| { + llm_prefs.fetch_openai_models_from_endpoint(ctx); + }); } fn build_page(