From e5062ae43278589bdf223562fabe8cb09012c752 Mon Sep 17 00:00:00 2001 From: Ryan Ward Date: Wed, 22 Jul 2026 15:58:55 -0500 Subject: [PATCH] feat: implement Crosscheck Work experiment Add the 'Crosscheck Work' experiment to the Agents settings. When enabled, a reviewer sub-agent is spawned after the main agent finishes a turn (with no pending tool calls). The reviewer critiques the output using a dedicated system prompt focused on correctness, simplicity, and code quality. If the reviewer does not respond with 'LGTM!', its feedback is injected as a synthetic user query back to the main agent, which must address it. This loop continues until the reviewer approves or max iterations is reached. Components: - Feature flag: CrosscheckWork (enabled in DOGFOOD_FLAGS) - Settings: agents.experiments.crosscheck_enabled, agents.experiments.crosscheck_model_id, agents.experiments.crosscheck_max_iterations - Settings UI: new 'Experiments' subpage under Agents - Crosscheck module: app/src/ai/crosscheck/ with prompt, reviewer model - Controller integration: hooks into AfterStreamFinished when no actions are queued, triggers reviewer, handles feedback injection - Provider support: OpenAI-compatible and Bedrock direct invocation - Safety: max iteration guard, error handling, reset on new user query --- app/src/ai/blocklist/controller.rs | 244 +++++++++++++++++ app/src/ai/crosscheck/mod.rs | 16 ++ app/src/ai/crosscheck/prompt.rs | 84 ++++++ app/src/ai/crosscheck/reviewer.rs | 404 +++++++++++++++++++++++++++++ app/src/ai/mod.rs | 1 + app/src/settings/ai.rs | 60 +++++ app/src/settings_view/ai_page.rs | 99 ++++++- app/src/settings_view/mod.rs | 4 + crates/galaxy_features/src/lib.rs | 7 + 9 files changed, 912 insertions(+), 7 deletions(-) create mode 100644 app/src/ai/crosscheck/mod.rs create mode 100644 app/src/ai/crosscheck/prompt.rs create mode 100644 app/src/ai/crosscheck/reviewer.rs diff --git a/app/src/ai/blocklist/controller.rs b/app/src/ai/blocklist/controller.rs index 4606120b..4e86f5f0 100644 --- a/app/src/ai/blocklist/controller.rs +++ b/app/src/ai/blocklist/controller.rs @@ -416,6 +416,8 @@ pub struct BlocklistAIController { Option, )>, >, + /// The crosscheck reviewer model for the "Crosscheck Work" experiment. + crosscheck_reviewer: ModelHandle, } enum InputQueryType { @@ -669,6 +671,16 @@ impl BlocklistAIController { OrchestrationEventStreamerEvent::ChildSpawned { .. } | OrchestrationEventStreamerEvent::ChildStatusChanged { .. } => {} }); + let crosscheck_reviewer = + ctx.add_model(crate::ai::crosscheck::CrosscheckReviewer::new); + ctx.subscribe_to_model(&crosscheck_reviewer, move |me, _, event, ctx| { + use crate::ai::crosscheck::{CrosscheckReviewerEvent, ReviewOutcome}; + let CrosscheckReviewerEvent::ReviewCompleted { + conversation_id, + outcome, + } = event; + me.handle_crosscheck_review_completed(*conversation_id, outcome.clone(), ctx); + }); Self { input_model, context_model, @@ -687,6 +699,7 @@ impl BlocklistAIController { pending_passive_suggestion_results: HashMap::new(), loop_detection: HashMap::new(), error_retry_counts: HashMap::new(), + crosscheck_reviewer, } } @@ -1239,6 +1252,10 @@ impl BlocklistAIController { // User sending a new query resets loop detection and error retry state — fresh context. self.loop_detection.remove(&conversation_id); self.error_retry_counts.remove(&conversation_id); + // Reset any in-flight crosscheck review for this conversation. + self.crosscheck_reviewer.update(ctx, |reviewer, _| { + reviewer.reset_review(conversation_id); + }); let is_viewer = self .terminal_model @@ -2140,6 +2157,225 @@ impl BlocklistAIController { /// Schedules an auto-resume-after-error for the conversation once the network is online /// and the auto-handoff sleep modal is closed, so the resume doesn't race the user's /// enable/dismiss decision on wake. + /// Handles the completion of a crosscheck review cycle. + /// + /// If the reviewer provided feedback, it is injected as a synthetic user + /// query to the main agent. If approved or max iterations reached, the + /// conversation is allowed to complete normally. + fn handle_crosscheck_review_completed( + &mut self, + conversation_id: AIConversationId, + outcome: crate::ai::crosscheck::ReviewOutcome, + ctx: &mut ModelContext, + ) { + use crate::ai::crosscheck::ReviewOutcome; + + match outcome { + ReviewOutcome::Approved => { + log::info!( + "[crosscheck] Work approved for conversation {conversation_id:?}" + ); + // Nothing to do — the conversation completes normally. + } + ReviewOutcome::MaxIterationsReached { last_feedback } => { + log::warn!( + "[crosscheck] Max iterations reached for {conversation_id:?}; showing last feedback to user" + ); + // Inject the last feedback as a visible message so the user is aware. + self.inject_crosscheck_feedback(conversation_id, last_feedback, true, ctx); + } + ReviewOutcome::Feedback { message } => { + log::info!( + "[crosscheck] Injecting reviewer feedback into conversation {conversation_id:?}" + ); + self.inject_crosscheck_feedback(conversation_id, message, false, ctx); + } + ReviewOutcome::Error { error } => { + log::error!( + "[crosscheck] Reviewer failed for {conversation_id:?}: {error}" + ); + // Don't block the conversation on reviewer errors; just log it. + } + } + } + + /// Injects crosscheck reviewer feedback as a synthetic user query to the + /// main agent, prompting it to address the feedback. + fn inject_crosscheck_feedback( + &mut self, + conversation_id: AIConversationId, + feedback: String, + is_final: bool, + ctx: &mut ModelContext, + ) { + let Some(conversation) = + BlocklistAIHistoryModel::as_ref(ctx).conversation(&conversation_id) + else { + return; + }; + let root_task_id = conversation.get_root_task_id().clone(); + + let prefix = if is_final { + "[CROSSCHECK REVIEWER - FINAL NOTE] The reviewer reached the maximum number of \ + review cycles. Below is the last feedback. Please address what you can, but you \ + may proceed even if not all items are resolved:" + } else { + "[CROSSCHECK REVIEWER] The following feedback was provided by an automated \ + reviewer. Please address ALL items below and then present your updated work:" + }; + + let corrective_msg = format!("{prefix}\n\n{feedback}"); + + let inputs = vec![AIAgentInput::UserQuery { + query: corrective_msg, + context: Arc::from([]), + static_query_type: None, + referenced_attachments: HashMap::new(), + user_query_mode: UserQueryMode::Normal, + running_command: None, + intended_agent: None, + }]; + + let _ = self.send_request_input( + RequestInput::for_task( + inputs, + root_task_id, + &self.active_session, + self.get_current_response_initiator(), + conversation_id, + self.terminal_surface_id, + ctx, + ), + None, + /*can_attempt_resume_on_error*/ false, + /*is_queued_prompt*/ false, + ctx, + ); + } + + /// Checks whether a crosscheck review should be triggered for a conversation + /// that just finished with no actions to queue (i.e., the agent is "done"). + /// + /// If eligible, extracts the agent's last output text and kicks off the + /// reviewer sub-agent. + fn maybe_trigger_crosscheck( + &mut self, + conversation_id: AIConversationId, + ctx: &mut ModelContext, + ) { + use settings::Setting; + + let ai_settings = crate::AISettings::as_ref(ctx); + if !ai_settings.is_crosscheck_enabled(ctx) { + return; + } + + // Don't trigger crosscheck for child conversations + let Some(conversation) = + BlocklistAIHistoryModel::as_ref(ctx).conversation(&conversation_id) + else { + return; + }; + if conversation.parent_conversation_id().is_some() { + return; + } + + // Extract the agent's last output text + let agent_output = self.extract_last_agent_output(conversation_id, ctx); + if agent_output.is_empty() { + return; + } + + // Determine which model to use for the reviewer + let model_id = { + let configured = ai_settings.crosscheck_model_id().to_string(); + if configured.is_empty() { + // Fall back to the conversation's active model + LLMPreferences::as_ref(ctx) + .get_active_base_model(ctx, Some(self.terminal_surface_id)) + .id + .to_string() + } else { + configured + } + }; + + let max_iterations = ai_settings.crosscheck_max_iterations(); + + self.crosscheck_reviewer.update(ctx, |reviewer, ctx| { + reviewer.start_review( + conversation_id, + max_iterations, + agent_output, + model_id, + ctx, + ); + }); + } + + /// Extracts the text content of the main agent's most recent output for a + /// conversation (used as input to the crosscheck reviewer). + fn extract_last_agent_output( + &self, + conversation_id: AIConversationId, + ctx: &AppContext, + ) -> String { + let Some(conversation) = + BlocklistAIHistoryModel::as_ref(ctx).conversation(&conversation_id) + else { + return String::new(); + }; + + // Get the last exchange's output messages + let exchanges = conversation.all_exchanges(); + let Some(last_exchange) = exchanges.last() else { + return String::new(); + }; + + let AIAgentOutputStatus::Finished { + finished_output: FinishedAIAgentOutput::Success { output }, + .. + } = &last_exchange.output_status + else { + return String::new(); + }; + + // Collect text messages from the output + output + .get() + .messages + .iter() + .filter_map(|msg| { + use crate::ai::agent::{AIAgentOutputMessageType, AIAgentTextSection}; + match &msg.message { + AIAgentOutputMessageType::Text(text) => { + let plain_text: String = text + .sections + .iter() + .filter_map(|section| match section { + AIAgentTextSection::PlainText { text } => { + Some(text.text().to_string()) + } + AIAgentTextSection::Code { code, .. } => { + Some(code.clone()) + } + _ => None, + }) + .collect::>() + .join("\n"); + if plain_text.is_empty() { + None + } else { + Some(plain_text) + } + } + _ => None, + } + }) + .collect::>() + .join("\n\n") + } + fn schedule_auto_resume_after_error( &mut self, conversation_id: AIConversationId, @@ -3467,6 +3703,14 @@ impl BlocklistAIController { ); }); } + + // Crosscheck Work experiment: if the main agent finished + // with no actions (i.e., it produced a final response), and + // this is not a child or passive conversation, trigger a + // reviewer sub-agent to critique the output. + if !is_child && !was_passive_request { + self.maybe_trigger_crosscheck(conversation_id, ctx); + } } // Cancelled streams will handle pending_response_stream updates synchronously. diff --git a/app/src/ai/crosscheck/mod.rs b/app/src/ai/crosscheck/mod.rs new file mode 100644 index 00000000..961a36ff --- /dev/null +++ b/app/src/ai/crosscheck/mod.rs @@ -0,0 +1,16 @@ +//! Crosscheck Work Experiment +//! +//! This module implements the "Crosscheck Work" experiment: after the main +//! agent finishes a turn (no pending tool calls), a reviewer sub-agent is +//! spawned to critique the output. If the reviewer does not respond with +//! "LGTM!", its feedback is injected as a synthetic user query to the main +//! agent, which must act on it. This loop continues until the reviewer is +//! satisfied or the maximum iteration count is reached. +//! +//! The reviewer is a lightweight, no-tool agent that only receives the +//! conversation context and produces text feedback. + +mod prompt; +mod reviewer; + +pub use reviewer::{CrosscheckReviewer, CrosscheckReviewerEvent, ReviewOutcome}; \ No newline at end of file diff --git a/app/src/ai/crosscheck/prompt.rs b/app/src/ai/crosscheck/prompt.rs new file mode 100644 index 00000000..73b2a008 --- /dev/null +++ b/app/src/ai/crosscheck/prompt.rs @@ -0,0 +1,84 @@ +//! System prompt for the crosscheck reviewer agent. + +/// The system prompt given to the crosscheck reviewer sub-agent. +/// +/// This prompt instructs the reviewer to be a critical but constructive +/// code reviewer, looking for correctness issues, simplification +/// opportunities, and potential regressions. +pub const CROSSCHECK_REVIEWER_SYSTEM_PROMPT: &str = r#"You are a critical code reviewer embedded in an AI coding assistant pipeline. + +Your job is to review the work produced by another AI coding agent. You receive the conversation history including the user's request and the agent's response (code changes, explanations, commands run, etc.). + +## Your Review Criteria + +1. **Correctness** — Are there bugs, logic errors, off-by-one mistakes, race conditions, or incorrect assumptions? +2. **Simplicity** — Can loops be simplified? Can algorithms be replaced with clearer alternatives? Is there unnecessary complexity? +3. **Completeness** — Did the agent fully address the user's request? Are there missed edge cases? +4. **Regressions** — Could the changes break existing functionality? +5. **Security** — Are there exposed secrets, injection risks, or unsafe patterns? +6. **Code Quality** — Are naming conventions followed? Is the code readable and idiomatic for the language? +7. **Optimization** — Are there obvious performance improvements (e.g., O(n²) where O(n) is possible)? + +## Your Response Format + +If the work is acceptable and you have no actionable feedback: +- Respond with exactly: LGTM! + +If you have feedback: +- List your concerns as numbered items, ordered by severity (most critical first). +- For each item, explain WHAT is wrong and suggest HOW to fix it. +- Be specific: reference file names, function names, or code snippets when possible. +- Be concise: don't repeat what the agent already knows. +- Focus on actionable improvements, not style nitpicks. + +## Important Rules + +- You have NO tools. You can only provide text feedback. +- Do NOT re-implement the solution. Only describe what should change. +- Do NOT provide feedback on things unrelated to the user's original request. +- If the response is non-code (e.g., an explanation or plan), review it for accuracy, completeness, and clarity. +- If the work is genuinely good, say "LGTM!" — do not invent problems. +- Be direct and respectful. The agent will automatically act on your feedback. +"#; + +/// The sentinel string that indicates the reviewer approves the work. +pub const LGTM_SENTINEL: &str = "LGTM!"; + +/// Checks whether a reviewer response indicates approval. +pub fn is_approved(response: &str) -> bool { + let trimmed = response.trim(); + // Accept the exact sentinel or the sentinel as the only meaningful content + trimmed == LGTM_SENTINEL + || trimmed.eq_ignore_ascii_case("lgtm!") + || trimmed.eq_ignore_ascii_case("lgtm") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_approved_exact() { + assert!(is_approved("LGTM!")); + } + + #[test] + fn test_is_approved_with_whitespace() { + assert!(is_approved(" LGTM! ")); + assert!(is_approved("\nLGTM!\n")); + } + + #[test] + fn test_is_approved_case_insensitive() { + assert!(is_approved("lgtm!")); + assert!(is_approved("Lgtm!")); + assert!(is_approved("lgtm")); + } + + #[test] + fn test_is_not_approved_with_feedback() { + assert!(!is_approved("LGTM! But also fix the typo.")); + assert!(!is_approved("1. Fix the loop\n2. Rename variable")); + assert!(!is_approved("")); + } +} \ No newline at end of file diff --git a/app/src/ai/crosscheck/reviewer.rs b/app/src/ai/crosscheck/reviewer.rs new file mode 100644 index 00000000..50c44262 --- /dev/null +++ b/app/src/ai/crosscheck/reviewer.rs @@ -0,0 +1,404 @@ +//! The crosscheck reviewer model. +//! +//! This model manages the lifecycle of a crosscheck review for a given +//! conversation. It tracks the current iteration, invokes the reviewer +//! LLM, and emits events to signal outcomes. + +use std::collections::HashMap; + +use galaxyui::{Entity, ModelContext, SingletonEntity}; +use settings::Setting; + +use super::prompt; +use crate::ai::agent::conversation::AIConversationId; +use crate::ai::llms::LLMPreferences; +use crate::ai::openai::client::{OpenAIClient, OpenAIClientConfig}; +use crate::ai::provider::ProviderConfig; +use crate::AISettings; + +/// Maximum default iterations if the setting is somehow zero. +const FALLBACK_MAX_ITERATIONS: u32 = 3; + +/// Max output tokens for the reviewer response. +const REVIEWER_MAX_OUTPUT_TOKENS: u32 = 4096; + +/// The outcome of a single crosscheck review cycle. +#[derive(Debug, Clone)] +pub enum ReviewOutcome { + /// The reviewer approved the work. + Approved, + /// The reviewer has feedback that should be sent to the main agent. + Feedback { message: String }, + /// The maximum number of iterations was reached; auto-accepting. + MaxIterationsReached { last_feedback: String }, + /// The reviewer invocation failed (e.g., network error). + Error { error: String }, +} + +/// Events emitted by the `CrosscheckReviewer`. +#[derive(Debug, Clone)] +pub enum CrosscheckReviewerEvent { + /// A review cycle completed with the given outcome. + ReviewCompleted { + conversation_id: AIConversationId, + outcome: ReviewOutcome, + }, +} + +/// Per-conversation crosscheck state. +#[derive(Debug, Clone)] +struct ConversationReviewState { + iteration: u32, + max_iterations: u32, +} + +/// Singleton model that manages crosscheck reviews across conversations. +pub struct CrosscheckReviewer { + /// Active review states keyed by conversation ID. + active_reviews: HashMap, +} + +impl Entity for CrosscheckReviewer { + type Event = CrosscheckReviewerEvent; +} + +impl CrosscheckReviewer { + pub fn new(_ctx: &mut ModelContext) -> Self { + Self { + active_reviews: HashMap::new(), + } + } + + /// Starts or continues a crosscheck review for the given conversation. + /// + /// `agent_output` is the text content of the main agent's last response + /// that should be reviewed. + /// + /// Returns `true` if a review was initiated, `false` if the conversation + /// is not eligible (e.g., already at max iterations). + pub fn start_review( + &mut self, + conversation_id: AIConversationId, + max_iterations: u32, + agent_output: String, + model_id: String, + ctx: &mut ModelContext, + ) -> bool { + let max_iterations = if max_iterations == 0 { + FALLBACK_MAX_ITERATIONS + } else { + max_iterations + }; + + let state = self + .active_reviews + .entry(conversation_id) + .or_insert_with(|| ConversationReviewState { + iteration: 0, + max_iterations, + }); + + if state.iteration >= state.max_iterations { + // Already exhausted iterations + return false; + } + + state.iteration += 1; + let current_iteration = state.iteration; + let max = state.max_iterations; + + log::info!( + "[crosscheck] Starting review iteration {current_iteration}/{max} for conversation {conversation_id:?} using model {model_id}" + ); + + // Resolve the provider config for the reviewer model + let provider_config = Self::resolve_reviewer_provider_config(&model_id, ctx); + + // Spawn the reviewer request asynchronously + let conversation_id_for_task = conversation_id; + ctx.spawn( + Self::invoke_reviewer(agent_output, model_id, provider_config), + move |me, reviewer_response, ctx| { + me.handle_reviewer_response( + conversation_id_for_task, + reviewer_response, + current_iteration, + max, + ctx, + ); + }, + ); + + true + } + + /// Clears crosscheck state for a conversation (e.g., on new user query or cancel). + pub fn reset_review(&mut self, conversation_id: AIConversationId) { + self.active_reviews.remove(&conversation_id); + } + + /// Returns the current iteration count for a conversation, or 0 if not active. + pub fn current_iteration(&self, conversation_id: AIConversationId) -> u32 { + self.active_reviews + .get(&conversation_id) + .map_or(0, |s| s.iteration) + } + + /// Returns `true` if a crosscheck review is in-flight for the conversation. + pub fn is_reviewing(&self, conversation_id: AIConversationId) -> bool { + self.active_reviews.contains_key(&conversation_id) + } + + /// Resolves the provider configuration for the reviewer model. + /// + /// Checks OpenAI-compatible routing first, then falls back to Bedrock. + fn resolve_reviewer_provider_config( + model_id: &str, + ctx: &ModelContext, + ) -> ProviderConfig { + let settings = AISettings::as_ref(ctx); + + // Check if this model has an OpenAI-compatible routing entry + if *settings.openai_enabled.value() { + let llm_prefs = LLMPreferences::as_ref(ctx); + if let Some(client_config) = llm_prefs.openai_client_config_for_model(model_id) { + return ProviderConfig::OpenAI(OpenAIClientConfig { + base_url: client_config.base_url.clone(), + api_key: client_config.api_key.clone(), + model: Some(model_id.to_string()), + max_input_tokens: client_config.max_input_tokens, + max_output_tokens: Some(REVIEWER_MAX_OUTPUT_TOKENS), + }); + } + } + + // Fall back to Bedrock via external config + if *settings.bedrock_enabled.value() { + let auth_method = *settings.bedrock_auth_method.value(); + let api_key_manager = ::ai::api_keys::ApiKeyManager::as_ref(ctx); + let mut config = crate::ai::bedrock::client::BedrockClientConfig { + auth_method, + profile: settings.bedrock_profile.value().clone(), + region: settings.bedrock_region.value().clone(), + access_key_id: settings.bedrock_access_key_id.value().clone(), + secret_access_key: settings.bedrock_secret_access_key.value().clone(), + session_token: None, + cross_region_inference: *settings.bedrock_cross_region_inference.value(), + }; + + if let ::ai::api_keys::AwsCredentialsState::Loaded { credentials, .. } = + api_key_manager.aws_credentials_state() + { + config.auth_method = crate::settings::BedrockAuthMethod::StaticKeys; + config.access_key_id = credentials.access_key().to_string(); + config.secret_access_key = credentials.secret_key().to_string(); + config.session_token = credentials.session_token().map(|s| s.to_string()); + } + + return ProviderConfig::Bedrock(config.with_external_fallbacks()); + } + + ProviderConfig::None + } + + /// Invokes the reviewer LLM with the given agent output. + /// + /// Sends a request to the configured provider with: + /// - System prompt: the crosscheck reviewer prompt + /// - User message: the agent's output to review + /// - No tools (reviewer is text-only) + /// - Non-streaming (collects the full response) + async fn invoke_reviewer( + agent_output: String, + model_id: String, + provider_config: ProviderConfig, + ) -> Result { + match provider_config { + ProviderConfig::OpenAI(config) => { + Self::invoke_via_openai(agent_output, model_id, config).await + } + ProviderConfig::Bedrock(_config) => { + // For Bedrock, we route through the OpenAI-compatible interface + // if available, otherwise use a simplified Bedrock invocation. + // Since most setups use LiteLLM or have OpenAI-compatible proxies, + // we log a warning and attempt Bedrock converse API. + Self::invoke_via_bedrock(agent_output, model_id, _config).await + } + ProviderConfig::None => Err( + "No AI provider configured. Enable either OpenAI/LiteLLM or Bedrock in settings." + .to_string(), + ), + } + } + + /// Invokes the reviewer via an OpenAI-compatible endpoint (non-streaming). + async fn invoke_via_openai( + agent_output: String, + model_id: String, + config: OpenAIClientConfig, + ) -> Result { + let client = OpenAIClient::from_config(config.clone()); + + let effective_model = config.model.unwrap_or(model_id); + + let request_body = serde_json::json!({ + "model": effective_model, + "messages": [ + { + "role": "system", + "content": prompt::CROSSCHECK_REVIEWER_SYSTEM_PROMPT + }, + { + "role": "user", + "content": format!( + "Please review the following agent output:\n\n---\n\n{agent_output}" + ) + } + ], + "max_tokens": REVIEWER_MAX_OUTPUT_TOKENS, + "stream": false + }); + + // Use a non-streaming request by calling the same endpoint but with stream: false + let url = format!( + "{}/chat/completions", + config.base_url.trim_end_matches('/') + ); + + let http = reqwest::Client::new(); + let mut request_builder = http.post(&url).json(&request_body); + + if let Some(ref key) = config.api_key { + request_builder = request_builder.header("Authorization", format!("Bearer {key}")); + } + + let response = request_builder + .send() + .await + .map_err(|e| format!("Crosscheck reviewer request failed: {e}"))?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(format!( + "Crosscheck reviewer returned HTTP {status}: {body}" + )); + } + + let body: serde_json::Value = response + .json() + .await + .map_err(|e| format!("Failed to parse reviewer response: {e}"))?; + + // Extract the assistant's message content from the OpenAI response format + let content = body["choices"][0]["message"]["content"] + .as_str() + .unwrap_or("") + .to_string(); + + if content.is_empty() { + return Err("Crosscheck reviewer returned an empty response.".to_string()); + } + + log::info!( + "[crosscheck] Reviewer response (len={}): {}", + content.len(), + &content[..content.len().min(200)] + ); + + Ok(content) + } + + /// Invokes the reviewer via AWS Bedrock Converse API. + async fn invoke_via_bedrock( + agent_output: String, + model_id: String, + config: crate::ai::bedrock::client::BedrockClientConfig, + ) -> Result { + use crate::ai::bedrock::client::BedrockClient; + use crate::ai::provider::types::{ConversationMessage, MessageContent, MessageRole}; + + let cross_region_inference = config.cross_region_inference; + let client = BedrockClient::from_config(config) + .await + .map_err(|e| format!("Failed to create Bedrock client: {e}"))?; + + let messages = vec![ConversationMessage { + role: MessageRole::User, + content: MessageContent::Text(format!( + "Please review the following agent output:\n\n---\n\n{agent_output}" + )), + }]; + + let system_prompt = Some(prompt::CROSSCHECK_REVIEWER_SYSTEM_PROMPT.to_string()); + + let (response_text, _input_tokens, _output_tokens) = client + .converse_collect( + &model_id, + messages, + system_prompt, + REVIEWER_MAX_OUTPUT_TOKENS as i32, + cross_region_inference, + ) + .await + .map_err(|e| format!("Bedrock reviewer invocation failed: {e}"))?; + + if response_text.is_empty() { + return Err("Crosscheck reviewer returned an empty response.".to_string()); + } + + log::info!( + "[crosscheck] Bedrock reviewer response (len={}): {}", + response_text.len(), + &response_text[..response_text.len().min(200)] + ); + + Ok(response_text) + } + + fn handle_reviewer_response( + &mut self, + conversation_id: AIConversationId, + response: Result, + current_iteration: u32, + max_iterations: u32, + ctx: &mut ModelContext, + ) { + let outcome = match response { + Err(error) => { + log::error!( + "[crosscheck] Reviewer invocation failed for {conversation_id:?}: {error}" + ); + self.active_reviews.remove(&conversation_id); + ReviewOutcome::Error { error } + } + Ok(text) => { + if prompt::is_approved(&text) { + log::info!( + "[crosscheck] Reviewer approved work for conversation {conversation_id:?} at iteration {current_iteration}" + ); + self.active_reviews.remove(&conversation_id); + ReviewOutcome::Approved + } else if current_iteration >= max_iterations { + log::warn!( + "[crosscheck] Max iterations ({max_iterations}) reached for conversation {conversation_id:?}; auto-accepting" + ); + self.active_reviews.remove(&conversation_id); + ReviewOutcome::MaxIterationsReached { + last_feedback: text, + } + } else { + log::info!( + "[crosscheck] Reviewer has feedback for conversation {conversation_id:?} (iteration {current_iteration}/{max_iterations})" + ); + ReviewOutcome::Feedback { message: text } + } + } + }; + + ctx.emit(CrosscheckReviewerEvent::ReviewCompleted { + conversation_id, + outcome, + }); + } +} diff --git a/app/src/ai/mod.rs b/app/src/ai/mod.rs index c1aa7a06..3b8af948 100644 --- a/app/src/ai/mod.rs +++ b/app/src/ai/mod.rs @@ -22,6 +22,7 @@ pub mod bedrock; pub(crate) mod bedrock_credentials; pub(crate) mod block_context; pub(crate) mod blocklist; +pub(crate) mod crosscheck; #[cfg(any(feature = "local_fs", not(target_family = "wasm")))] pub(crate) mod codebase_auto_indexing; pub mod control_code_parser; diff --git a/app/src/settings/ai.rs b/app/src/settings/ai.rs index a2416a14..2913cc4e 100644 --- a/app/src/settings/ai.rs +++ b/app/src/settings/ai.rs @@ -1865,6 +1865,48 @@ define_settings_group!(AISettings, settings: [ sync_to_cloud: SyncToCloud::Globally(RespectUserSyncSetting::No), private: true, } + + // Whether the "Crosscheck Work" experiment is enabled. + // When enabled, a reviewer sub-agent is spawned after the main agent finishes + // a turn (with no pending tool calls) to critique the output. The feedback + // loop continues until the reviewer responds with "LGTM!". + crosscheck_enabled: CrosscheckEnabled { + type: bool, + default: false, + supported_platforms: SupportedPlatforms::ALL, + sync_to_cloud: SyncToCloud::Globally(RespectUserSyncSetting::Yes), + private: false, + toml_path: "agents.experiments.crosscheck_enabled", + description: "Enables the Crosscheck Work experiment: a reviewer agent critiques agent output until satisfied.", + feature_flag: FeatureFlag::CrosscheckWork, + } + + // The model ID to use for the crosscheck reviewer agent. + // Must be a valid model ID from the configured LLM providers. + // If empty, the currently active model for the conversation is used. + crosscheck_model_id: CrosscheckModelId { + type: String, + default: String::new(), + supported_platforms: SupportedPlatforms::ALL, + sync_to_cloud: SyncToCloud::Globally(RespectUserSyncSetting::Yes), + private: false, + toml_path: "agents.experiments.crosscheck_model_id", + description: "The LLM model ID used by the crosscheck reviewer agent.", + feature_flag: FeatureFlag::CrosscheckWork, + } + + // Maximum number of crosscheck review cycles before stopping. + // Prevents infinite feedback loops between the main agent and reviewer. + crosscheck_max_iterations: CrosscheckMaxIterations { + type: u32, + default: 3, + supported_platforms: SupportedPlatforms::ALL, + sync_to_cloud: SyncToCloud::Globally(RespectUserSyncSetting::Yes), + private: false, + toml_path: "agents.experiments.crosscheck_max_iterations", + description: "Maximum number of crosscheck review cycles before auto-accepting.", + feature_flag: FeatureFlag::CrosscheckWork, + } ]); impl AISettings { @@ -2049,6 +2091,24 @@ impl AISettings { /// Returns true when local-to-cloud handoff is effectively enabled. /// False when the user/org has disabled it, cloud conversations are off, /// or AI is globally off. + /// Returns `true` when the Crosscheck Work experiment is active. + pub fn is_crosscheck_enabled(&self, app: &galaxyui::AppContext) -> bool { + self.is_any_ai_enabled(app) + && FeatureFlag::CrosscheckWork.is_enabled() + && *self.crosscheck_enabled + } + + /// Returns the model ID configured for the crosscheck reviewer, or empty + /// string if the active model should be used. + pub fn crosscheck_model_id(&self) -> &str { + &self.crosscheck_model_id + } + + /// Maximum number of crosscheck review iterations. + pub fn crosscheck_max_iterations(&self) -> u32 { + (*self.crosscheck_max_iterations).max(1) + } + pub fn is_cloud_handoff_enabled(&self, app: &galaxyui::AppContext) -> bool { if !self.is_any_ai_enabled(app) || *self.should_force_disable_cloud_handoff { return false; diff --git a/app/src/settings_view/ai_page.rs b/app/src/settings_view/ai_page.rs index 5ec492f6..c9630705 100644 --- a/app/src/settings_view/ai_page.rs +++ b/app/src/settings_view/ai_page.rs @@ -92,13 +92,14 @@ use crate::settings::{ AgentModeCodingPermissionsType, AgentModeCommandExecutionDenylist, AgentModeCommandExecutionPredicate, AgentModeQuerySuggestionsEnabled, BedrockAutoLogin, BedrockEnabled, CanUseWarpCreditsForFallback, CodeSettings, CodebaseContextEnabled, - FileBasedMcpEnabled, GitOperationsAutogenEnabled, IncludeAgentCommandsInHistory, InputSettings, - IntelligentAutosuggestionsEnabled, LongRunningCommandSubmissionMode, MemoryEnabled, - NLDInTerminalEnabled, NaturalLanguageAutosuggestionsEnabled, OpenAIEnabled, - OrchestrationMessageDisplayMode, PromptSubmissionMode, RuleSuggestionsEnabled, - SharedBlockTitleGenerationEnabled, ShouldRenderCLIAgentToolbar, - ShouldRenderUseAgentToolbarForUserCommands, ShowAgentTips, ShowConversationHistory, - ShowHintText, ThinkingDisplayMode, VoiceInputEnabled, WarpDriveContextEnabled, + CrosscheckEnabled, FileBasedMcpEnabled, GitOperationsAutogenEnabled, + IncludeAgentCommandsInHistory, InputSettings, IntelligentAutosuggestionsEnabled, + LongRunningCommandSubmissionMode, MemoryEnabled, NLDInTerminalEnabled, + NaturalLanguageAutosuggestionsEnabled, OpenAIEnabled, OrchestrationMessageDisplayMode, + PromptSubmissionMode, RuleSuggestionsEnabled, SharedBlockTitleGenerationEnabled, + ShouldRenderCLIAgentToolbar, ShouldRenderUseAgentToolbarForUserCommands, ShowAgentTips, + ShowConversationHistory, ShowHintText, ThinkingDisplayMode, VoiceInputEnabled, + WarpDriveContextEnabled, }; use crate::terminal::session_settings::{SessionSettings, SessionSettingsChangedEvent}; use crate::terminal::CLIAgent; @@ -128,6 +129,8 @@ pub enum AISubpage { Bedrock, /// OpenAI-compatible (LiteLLM) provider configuration. OpenAI, + /// Experimental features. + Experiments, } impl AISubpage { @@ -139,6 +142,7 @@ impl AISubpage { SettingsSection::ThirdPartyCLIAgents => Some(Self::ThirdPartyCLIAgents), SettingsSection::Bedrock => Some(Self::Bedrock), SettingsSection::OpenAI => Some(Self::OpenAI), + SettingsSection::Experiments => Some(Self::Experiments), // AgentMCPServers renders the standalone MCPServers page, not an AI subpage. _ => None, } @@ -2866,6 +2870,9 @@ impl AISettingsPageView { let title: Option<&str> = None; return (PageType::new_uncategorized(widgets, title), None); } + Some(AISubpage::Experiments) => { + widgets.push(Box::new(ExperimentsWidget::default())); + } } // Subpage widgets render their own subheader-sized titles internally, @@ -3656,6 +3663,7 @@ pub enum AISettingsPageAction { agent: Option, }, ToggleCloudAgentComputerUse, + ToggleCrosscheckEnabled, } impl From<&AISettingsPageAction> for LoginGatedFeature { @@ -4356,6 +4364,12 @@ impl TypedActionView for AISettingsPageView { }); ctx.notify(); } + AISettingsPageAction::ToggleCrosscheckEnabled => { + AISettings::handle(ctx).update(ctx, |settings, ctx| { + report_if_error!(settings.crosscheck_enabled.toggle_and_save_value(ctx)); + }); + ctx.notify(); + } AISettingsPageAction::ToggleBedrockEnabled => { AISettings::handle(ctx).update(ctx, |settings, ctx| { report_if_error!(settings.bedrock_enabled.toggle_and_save_value(ctx)); @@ -9790,6 +9804,77 @@ impl SettingsWidget for CustomModelRoutersWidget { } } +// ── Experiments widget ────────────────────────────────────────────────────── + +#[derive(Default)] +struct ExperimentsWidget { + crosscheck_toggle: SwitchStateHandle, +} + +impl SettingsWidget for ExperimentsWidget { + type View = AISettingsPageView; + + fn search_terms(&self) -> &str { + "experiments crosscheck review reviewer sub-agent feedback loop lgtm" + } + + fn should_render(&self, _app: &AppContext) -> bool { + FeatureFlag::CrosscheckWork.is_enabled() + } + + fn render( + &self, + view: &Self::View, + appearance: &Appearance, + app: &AppContext, + ) -> Box { + let ai_settings = AISettings::as_ref(app); + let is_any_ai_enabled = ai_settings.is_any_ai_enabled(app); + + let header = build_sub_header( + appearance, + "Experiments", + Some(styles::header_font_color(is_any_ai_enabled, app)), + ) + .with_margin_bottom(HEADER_PADDING) + .finish(); + + let crosscheck_toggle = render_ai_setting_toggle::( + "Crosscheck Work", + AISettingsPageAction::ToggleCrosscheckEnabled, + *ai_settings.crosscheck_enabled, + is_any_ai_enabled, + self.crosscheck_toggle.clone(), + &view.local_only_icon_tooltip_states, + app, + ); + + let crosscheck_description = render_ai_setting_description( + "When enabled, a reviewer agent critiques the main agent's output after each turn. \ + The feedback loop continues until the reviewer responds with \"LGTM!\" or the \ + maximum iteration count is reached.", + is_any_ai_enabled, + app, + ); + + let model_description = render_ai_setting_description( + "Configure the reviewer model and max iterations in settings.toml under \ + [agents.experiments].", + is_any_ai_enabled, + app, + ); + + let column = Flex::column() + .with_child(header) + .with_child(crosscheck_toggle) + .with_child(crosscheck_description) + .with_child(model_description) + .finish(); + + column + } +} + mod styles { use galaxy_core::ui::appearance::Appearance; use galaxy_core::ui::theme::Fill; diff --git a/app/src/settings_view/mod.rs b/app/src/settings_view/mod.rs index cb467711..ccfbe9ac 100644 --- a/app/src/settings_view/mod.rs +++ b/app/src/settings_view/mod.rs @@ -272,6 +272,7 @@ pub enum SettingsSection { ThirdPartyCLIAgents, Bedrock, OpenAI, + Experiments, /// Internal backing-page identifier for CodeSettingsPageView. Multiple subpages /// (CodeIndexing, EditorAndCodeReview) share this single backing page, /// so this variant is needed as the key in `settings_pages`. @@ -306,6 +307,7 @@ impl Display for SettingsSection { SettingsSection::ThirdPartyCLIAgents => write!(f, "Third party CLI agents"), SettingsSection::Bedrock => write!(f, "AWS Bedrock"), SettingsSection::OpenAI => write!(f, "OpenAI / LiteLLM"), + SettingsSection::Experiments => write!(f, "Experiments"), SettingsSection::Warpify => write!(f, "Wormhole"), SettingsSection::CodeIndexing => write!(f, "Indexing and projects"), SettingsSection::EditorAndCodeReview => write!(f, "Editor and Code Review"), @@ -363,6 +365,7 @@ impl SettingsSection { Self::ThirdPartyCLIAgents, Self::Bedrock, Self::OpenAI, + Self::Experiments, ] } @@ -405,6 +408,7 @@ impl FromStr for SettingsSection { "CloudEnvironments" | "Environments" => Ok(Self::CloudEnvironments), "OzCloudAPIKeys" => Ok(Self::OzCloudAPIKeys), "SharedBlocks" | "Shared Blocks" => Ok(Self::SharedBlocks), + "Experiments" => Ok(Self::Experiments), _ => Err(()), } } diff --git a/crates/galaxy_features/src/lib.rs b/crates/galaxy_features/src/lib.rs index 69315d87..93af89b4 100644 --- a/crates/galaxy_features/src/lib.rs +++ b/crates/galaxy_features/src/lib.rs @@ -910,6 +910,12 @@ pub enum FeatureFlag { LspCompletion, LspRename, LspSignatureHelp, + + /// Enables the "Crosscheck Work" experiment, which spawns a reviewer + /// sub-agent after the main agent finishes a turn. The reviewer critiques + /// the output and sends feedback back to the main agent until it responds + /// with "LGTM!". + CrosscheckWork, } static FLAG_STATES: [AtomicBool; cardinality::()] = @@ -981,6 +987,7 @@ pub const DOGFOOD_FLAGS: &[FeatureFlag] = &[ FeatureFlag::ContextWindowUsageBreakdown, FeatureFlag::CloudRunners, FeatureFlag::WaitForEventsParentRegistration, + FeatureFlag::CrosscheckWork, ]; /// Features enabled for feature preview build users (e.g.: Friends of Warp).