Initial public release of Warp.
Repo-Sync-Origin: warpdotdev/warp-internal@12af1d983b
This commit is contained in:
David Stern
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
use crate::integration_testing::agent_mode::util::get_base_server_url;
|
||||
use anyhow::Result;
|
||||
use reqwest::blocking::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct LLMGenerateRequest {
|
||||
pub prompt: String,
|
||||
pub user_messages: Vec<String>,
|
||||
/// These are model IDs internal to warp-server.
|
||||
/// See warp-server/logic/ai/llm/llm.go
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ExactTokenUsage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_reads: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_writes: Option<i32>,
|
||||
pub total_input: i32,
|
||||
pub output: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct LLMGenerateResponse {
|
||||
pub content: String,
|
||||
pub token_usage: ExactTokenUsage,
|
||||
}
|
||||
|
||||
pub fn generate_llm_response(
|
||||
client: &Client,
|
||||
request: LLMGenerateRequest,
|
||||
) -> Result<LLMGenerateResponse> {
|
||||
let url = format!("{}/agent-mode-evals/llm_generate", get_base_server_url());
|
||||
|
||||
let response = client.post(&url).json(&request).send()?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Failed to generate LLM response: {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
let response = response.json::<LLMGenerateResponse>()?;
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
pub mod llm_generate;
|
||||
|
||||
use anyhow::Result;
|
||||
use llm_generate::LLMGenerateRequest;
|
||||
use reqwest::blocking::Client;
|
||||
use serde::Deserialize;
|
||||
use warp_multi_agent_api::{
|
||||
apply_file_diffs_result::success::UpdatedFileContent, message, Message,
|
||||
};
|
||||
|
||||
use crate::ai::agent::conversation::AIConversation;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LLMJudgeConfig {
|
||||
pub prompt: String,
|
||||
pub model: &'static str,
|
||||
}
|
||||
|
||||
/// The LLM judge's response content will be directly unmarshalled into this struct.
|
||||
/// Each judge's prompt must instruct the LLM to output its response in this format.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct LLMJudgeResult {
|
||||
pub pass: bool,
|
||||
pub critique: String,
|
||||
}
|
||||
|
||||
pub struct LLMJudge {
|
||||
config: LLMJudgeConfig,
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl LLMJudge {
|
||||
pub fn new(config: LLMJudgeConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
client: Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format the conversation history to pass to the LLM judge.
|
||||
/// TODO: support optional different formatting for different judges
|
||||
fn format_conversation_history_for_llm_judge(conversation: &AIConversation) -> Result<String> {
|
||||
let messages = conversation.all_linearized_messages();
|
||||
|
||||
// Filter out tool call result contents from the task's messages.
|
||||
let filtered_messages = filter_tool_call_results_from_messages(messages.into_iter());
|
||||
|
||||
// Use debug format for now since we can't serialize protos to JSON.
|
||||
let task_json = format!("{filtered_messages:?}");
|
||||
|
||||
Ok(task_json)
|
||||
}
|
||||
|
||||
pub fn judge(&self, conversation: &AIConversation) -> Result<LLMJudgeResult> {
|
||||
let formatted_conversation_history =
|
||||
Self::format_conversation_history_for_llm_judge(conversation)?;
|
||||
let request = LLMGenerateRequest {
|
||||
prompt: self.config.prompt.to_owned(),
|
||||
user_messages: vec![formatted_conversation_history],
|
||||
model_id: self.config.model.to_owned(),
|
||||
};
|
||||
|
||||
let response = llm_generate::generate_llm_response(&self.client, request)?;
|
||||
|
||||
// Parse the response content as JSON
|
||||
let result: LLMJudgeResult = serde_json::from_str(&response.content)?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter out tool call result contents while preserving structure and success/error status
|
||||
pub fn filter_tool_call_result(result: &message::ToolCallResult) -> message::ToolCallResult {
|
||||
use message::tool_call_result::Result as ToolResult;
|
||||
use warp_multi_agent_api::*;
|
||||
|
||||
let filtered_result = match &result.result {
|
||||
Some(ToolResult::RunShellCommand(cmd_result)) =>
|
||||
{
|
||||
#[allow(deprecated)]
|
||||
Some(ToolResult::RunShellCommand(RunShellCommandResult {
|
||||
command: cmd_result.command.clone(),
|
||||
output: Default::default(),
|
||||
exit_code: Default::default(),
|
||||
result: Some(
|
||||
warp_multi_agent_api::run_shell_command_result::Result::CommandFinished(
|
||||
warp_multi_agent_api::ShellCommandFinished {
|
||||
command_id: "command_id".to_string(),
|
||||
output: "[OUTPUT OMITTED]".to_string(),
|
||||
exit_code: cmd_result.exit_code,
|
||||
},
|
||||
),
|
||||
),
|
||||
}))
|
||||
}
|
||||
Some(ToolResult::ReadFiles(read_result)) => {
|
||||
use read_files_result::Result as ReadResult;
|
||||
let filtered_read_result = match &read_result.result {
|
||||
Some(ReadResult::TextFilesSuccess(success)) => {
|
||||
// Keep file paths and line ranges but remove content
|
||||
let filtered_files = success
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| FileContent {
|
||||
file_path: file.file_path.clone(),
|
||||
line_range: file.line_range,
|
||||
content: "[CONTENT OMITTED]".to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(ReadResult::TextFilesSuccess(
|
||||
read_files_result::TextFilesSuccess {
|
||||
files: filtered_files,
|
||||
},
|
||||
))
|
||||
}
|
||||
Some(ReadResult::AnyFilesSuccess(success)) => {
|
||||
// Keep file paths and line ranges but remove content
|
||||
let filtered_files = success
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| match &file.content {
|
||||
Some(any_file_content::Content::TextContent(text_content)) => {
|
||||
AnyFileContent {
|
||||
content: Some(any_file_content::Content::TextContent(
|
||||
warp_multi_agent_api::FileContent {
|
||||
file_path: text_content.file_path.clone(),
|
||||
content: "[CONTENT OMITTED]".to_string(),
|
||||
line_range: text_content.line_range,
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
Some(any_file_content::Content::BinaryContent(binary_content)) => {
|
||||
AnyFileContent {
|
||||
content: Some(any_file_content::Content::BinaryContent(
|
||||
warp_multi_agent_api::BinaryFileContent {
|
||||
file_path: binary_content.file_path.clone(),
|
||||
data: vec![],
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
None => unreachable!(
|
||||
"AnyFileContent should always contain TextContent or BinaryContent"
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
Some(ReadResult::AnyFilesSuccess(
|
||||
read_files_result::AnyFilesSuccess {
|
||||
files: filtered_files,
|
||||
},
|
||||
))
|
||||
}
|
||||
Some(ReadResult::Error(err)) => Some(ReadResult::Error(err.clone())),
|
||||
None => None,
|
||||
};
|
||||
Some(ToolResult::ReadFiles(ReadFilesResult {
|
||||
result: filtered_read_result,
|
||||
}))
|
||||
}
|
||||
Some(ToolResult::SearchCodebase(search_result)) => {
|
||||
use search_codebase_result::Result as SearchResult;
|
||||
let filtered_search_result = match &search_result.result {
|
||||
Some(SearchResult::Success(success)) => {
|
||||
// Keep file paths and line ranges but remove file contents
|
||||
let filtered_files = success
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| FileContent {
|
||||
file_path: file.file_path.clone(),
|
||||
line_range: file.line_range,
|
||||
content: "[CONTENT OMITTED]".to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(SearchResult::Success(search_codebase_result::Success {
|
||||
files: filtered_files,
|
||||
}))
|
||||
}
|
||||
Some(SearchResult::Error(err)) => Some(SearchResult::Error(err.clone())),
|
||||
None => None,
|
||||
};
|
||||
Some(ToolResult::SearchCodebase(SearchCodebaseResult {
|
||||
result: filtered_search_result,
|
||||
}))
|
||||
}
|
||||
Some(ToolResult::ApplyFileDiffs(diff_result)) => {
|
||||
use apply_file_diffs_result::Result as DiffResult;
|
||||
let filtered_diff_result = match &diff_result.result {
|
||||
Some(DiffResult::Success(success)) => {
|
||||
// Keep file paths and line ranges but remove file contents
|
||||
let filtered_files = success
|
||||
.updated_files_v2
|
||||
.iter()
|
||||
.map(|file| UpdatedFileContent {
|
||||
file: file.file.as_ref().map(|file_content| FileContent {
|
||||
file_path: file_content.file_path.clone(),
|
||||
line_range: file_content.line_range,
|
||||
content: "[CONTENT OMITTED]".to_string(),
|
||||
}),
|
||||
was_edited_by_user: file.was_edited_by_user,
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(DiffResult::Success(apply_file_diffs_result::Success {
|
||||
updated_files_v2: filtered_files,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Some(DiffResult::Error(err)) => Some(DiffResult::Error(err.clone())),
|
||||
None => None,
|
||||
};
|
||||
Some(ToolResult::ApplyFileDiffs(ApplyFileDiffsResult {
|
||||
result: filtered_diff_result,
|
||||
}))
|
||||
}
|
||||
Some(ToolResult::Grep(grep_result)) => {
|
||||
use grep_result::Result as GrepResult;
|
||||
let filtered_grep_result = match &grep_result.result {
|
||||
Some(GrepResult::Success(success)) => {
|
||||
// Keep only file paths, remove all matched content and line numbers
|
||||
let filtered_files = success
|
||||
.matched_files
|
||||
.iter()
|
||||
.map(|file| {
|
||||
grep_result::success::GrepFileMatch {
|
||||
file_path: file.file_path.clone(),
|
||||
// These can be very long so they're filtered, but we can't replace with a placeholder.
|
||||
// The prompt should address this.
|
||||
matched_lines: vec![],
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(GrepResult::Success(grep_result::Success {
|
||||
matched_files: filtered_files,
|
||||
}))
|
||||
}
|
||||
Some(GrepResult::Error(err)) => Some(GrepResult::Error(err.clone())),
|
||||
None => None,
|
||||
};
|
||||
Some(ToolResult::Grep(warp_multi_agent_api::GrepResult {
|
||||
result: filtered_grep_result,
|
||||
}))
|
||||
}
|
||||
other => other.clone(),
|
||||
};
|
||||
|
||||
message::ToolCallResult {
|
||||
tool_call_id: result.tool_call_id.clone(),
|
||||
context: None, // Remove context to reduce noise
|
||||
result: filtered_result,
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter out tool call result contents while preserving structure
|
||||
pub fn filter_tool_call_results_from_messages<'a>(
|
||||
messages: impl Iterator<Item = &'a Message>,
|
||||
) -> Vec<Message> {
|
||||
messages
|
||||
.map(|msg| {
|
||||
let mut filtered_msg = msg.clone();
|
||||
if let Some(message::Message::ToolCallResult(ref tool_result)) = &msg.message {
|
||||
filtered_msg.message = Some(message::Message::ToolCallResult(
|
||||
filter_tool_call_result(tool_result),
|
||||
));
|
||||
}
|
||||
filtered_msg
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
mod assertions;
|
||||
pub mod llm_judge;
|
||||
mod step;
|
||||
mod user_defaults;
|
||||
mod util;
|
||||
use std::collections::HashSet;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
|
||||
use crate::ai::agent::{AIAgentOutputStatus, FinishedAIAgentOutput};
|
||||
pub use crate::ai::blocklist::agent_view::AgentViewState;
|
||||
use crate::BlocklistAIHistoryModel;
|
||||
use crate::{ai::agent::AIAgentActionType, integration_testing::view_getters::terminal_view};
|
||||
pub use assertions::*;
|
||||
pub use step::*;
|
||||
pub use user_defaults::*;
|
||||
pub use util::*;
|
||||
use warpui::integration::PersistedDataMap;
|
||||
pub use warpui::integration::RUNTIME_TAG_FAILURE_REASON;
|
||||
use warpui::{App, SingletonEntity as _, WindowId};
|
||||
|
||||
pub const TOTAL_REQUEST_COST_PREFIX: &str = "Total request cost: ";
|
||||
pub const TOTAL_EXCHANGES_PREFIX: &str = "Total number of exchanges: ";
|
||||
pub const TOTAL_TOKEN_USAGE_PREFIX: &str = "Total token usage: ";
|
||||
|
||||
pub const RUNTIME_TAG_TOTAL_REQUEST_COST: &str = "total_request_cost";
|
||||
pub const RUNTIME_TAG_TOTAL_EXCHANGES: &str = "total_exchanges";
|
||||
pub const RUNTIME_TAG_TOKEN_USAGE_PREFIX: &str = "token_usage.";
|
||||
|
||||
const CODE_DIFF_OUTPUT_FILE_ENV_VAR: &str = "CODE_DIFF_OUTPUT_FILE";
|
||||
|
||||
pub fn output_code_diff_with_base_commit(
|
||||
base_commit: &str,
|
||||
working_dir: &str,
|
||||
test_files_str: &str,
|
||||
) {
|
||||
use command::blocking::Command;
|
||||
|
||||
let Some(mut output_file) = open_debug_file_from_env(CODE_DIFF_OUTPUT_FILE_ENV_VAR) else {
|
||||
log::error!("Could not open debug file from env");
|
||||
return;
|
||||
};
|
||||
// Clear the test files from the diff, because we are not interested in seeing those.
|
||||
log::debug!(
|
||||
"[GIT OPERATION] mod.rs output_code_diff_with_base_commit git checkout {base_commit} -- {test_files_str}"
|
||||
);
|
||||
let _ = Command::new("git")
|
||||
.args(["checkout", base_commit, "--", test_files_str])
|
||||
.current_dir(working_dir)
|
||||
.output();
|
||||
log::debug!(
|
||||
"[GIT OPERATION] mod.rs output_code_diff_with_base_commit git --no-pager diff {base_commit}"
|
||||
);
|
||||
let git_diff_output = Command::new("git")
|
||||
.args(["--no-pager", "diff", base_commit])
|
||||
.current_dir(working_dir)
|
||||
.output();
|
||||
write_git_diff_output_to_file(
|
||||
git_diff_output,
|
||||
&mut output_file,
|
||||
&std::env::var(CODE_DIFF_OUTPUT_FILE_ENV_VAR)
|
||||
.expect("Could not find diff output file env var"),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn output_code_diff_debug_info(app: &mut App, window_id: WindowId) {
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
let Some(current_dir) = terminal_view.read(app, |terminal_view, _| terminal_view.pwd()) else {
|
||||
log::error!("Could not get current directory");
|
||||
return;
|
||||
};
|
||||
BlocklistAIHistoryModel::handle(app).update(app, |history_model, _| {
|
||||
let Some(conversation) = history_model.active_conversation(terminal_view.id()) else {
|
||||
return;
|
||||
};
|
||||
let mut edited_files = HashSet::new();
|
||||
for exchange in conversation.all_exchanges().into_iter() {
|
||||
let AIAgentOutputStatus::Finished { finished_output } = &exchange.output_status else {
|
||||
continue;
|
||||
};
|
||||
if let FinishedAIAgentOutput::Success { output } = finished_output {
|
||||
let agent_output = output.get();
|
||||
for action in agent_output.actions() {
|
||||
if let AIAgentActionType::RequestFileEdits { file_edits, .. } = &action.action {
|
||||
for edit in file_edits {
|
||||
if let Some(file) = edit.file() {
|
||||
edited_files.insert(file.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut output_file = open_debug_file_from_env(CODE_DIFF_OUTPUT_FILE_ENV_VAR);
|
||||
if let Some(output_file) = &mut output_file {
|
||||
use command::blocking::Command;
|
||||
use std::io::Write;
|
||||
if edited_files.is_empty() {
|
||||
writeln!(output_file, "No files were edited for this test")
|
||||
.expect("Failed to write to code diff file");
|
||||
} else {
|
||||
for file_name in edited_files {
|
||||
log::debug!(
|
||||
"[GIT OPERATION] mod.rs output_code_diff_debug_info git diff -- {file_name}"
|
||||
);
|
||||
let output = Command::new("git")
|
||||
.args(["diff", "--", &file_name])
|
||||
.current_dir(¤t_dir)
|
||||
.output();
|
||||
write_git_diff_output_to_file(output, output_file, &file_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn write_git_diff_output_to_file(
|
||||
diff_output: std::io::Result<std::process::Output>,
|
||||
file: &mut File,
|
||||
file_name: &str,
|
||||
) {
|
||||
match diff_output {
|
||||
Ok(output) if output.status.success() => {
|
||||
let diff = String::from_utf8_lossy(&output.stdout);
|
||||
writeln!(file, "Diff for file: {file_name}\n{diff}\n")
|
||||
.expect("Failed to write diff to code diff file");
|
||||
}
|
||||
Ok(output) => {
|
||||
let err = String::from_utf8_lossy(&output.stderr);
|
||||
writeln!(
|
||||
file,
|
||||
"Failed to get diff for file: {file_name}\nGit error:\n{err}\n"
|
||||
)
|
||||
.expect("Failed to write error to code diff file");
|
||||
}
|
||||
Err(e) => {
|
||||
writeln!(
|
||||
file,
|
||||
"Failed to run git diff for file: {file_name}\nError: {e}\n"
|
||||
)
|
||||
.expect("Failed to write command error to code diff file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn output_conversation_debug_info(
|
||||
app: &mut App,
|
||||
window_id: WindowId,
|
||||
persisted_data: &mut PersistedDataMap,
|
||||
) {
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
BlocklistAIHistoryModel::handle(app).update(app, |history_model, _| {
|
||||
let Some(conversation) = history_model.active_conversation(terminal_view.id()) else {
|
||||
return;
|
||||
};
|
||||
let mut output_file = open_debug_file_from_env("DEBUG_OUTPUT_FILE");
|
||||
|
||||
// Create a function to handle output
|
||||
let mut write_to_debug_file = |text: &str| {
|
||||
if let Some(file) = &mut output_file {
|
||||
use std::io::Write;
|
||||
writeln!(file, "{text}").expect("Failed to write to debug output file");
|
||||
} else {
|
||||
println!("{text}");
|
||||
}
|
||||
};
|
||||
|
||||
// Add timestamp to the header
|
||||
let current_time = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ");
|
||||
write_to_debug_file(&format!(
|
||||
"========Conversation Debug Info (Generated: {current_time})========"
|
||||
));
|
||||
let debug_link = conversation
|
||||
.server_conversation_token()
|
||||
.map(|token| {
|
||||
// The debug link within the container will be using host.docker.internal, but we're opening
|
||||
// from outside the container.
|
||||
// The server is configured to always write debug data to GCS instead of locally when run for evals, so we replace
|
||||
// with staging.warp.dev instead of localhost:8080.
|
||||
token
|
||||
.debug_link()
|
||||
.replace("host.docker.internal:8080", "staging.warp.dev")
|
||||
})
|
||||
.unwrap_or("unavailable".to_owned());
|
||||
write_to_debug_file(&format!("Conversation Debug Link: {debug_link}"));
|
||||
|
||||
let total_request_cost = conversation.total_request_cost();
|
||||
let total_exchanges = conversation.all_exchanges().len();
|
||||
let token_usage = conversation.total_token_usage();
|
||||
|
||||
// Populate runtime tags with conversation data
|
||||
persisted_data.insert(
|
||||
RUNTIME_TAG_TOTAL_REQUEST_COST.to_string(),
|
||||
total_request_cost.to_string(),
|
||||
);
|
||||
persisted_data.insert(
|
||||
RUNTIME_TAG_TOTAL_EXCHANGES.to_string(),
|
||||
total_exchanges.to_string(),
|
||||
);
|
||||
|
||||
// Add token usage as separate runtime tags
|
||||
for usage in token_usage.iter() {
|
||||
persisted_data.insert(
|
||||
format!(
|
||||
"{}{}.total_input",
|
||||
RUNTIME_TAG_TOKEN_USAGE_PREFIX, usage.model_id
|
||||
),
|
||||
usage.total_input.to_string(),
|
||||
);
|
||||
persisted_data.insert(
|
||||
format!(
|
||||
"{}{}.output",
|
||||
RUNTIME_TAG_TOKEN_USAGE_PREFIX, usage.model_id
|
||||
),
|
||||
usage.output.to_string(),
|
||||
);
|
||||
persisted_data.insert(
|
||||
format!(
|
||||
"{}{}.input_cache_read",
|
||||
RUNTIME_TAG_TOKEN_USAGE_PREFIX, usage.model_id
|
||||
),
|
||||
usage.input_cache_read.to_string(),
|
||||
);
|
||||
persisted_data.insert(
|
||||
format!(
|
||||
"{}{}.input_cache_write",
|
||||
RUNTIME_TAG_TOKEN_USAGE_PREFIX, usage.model_id
|
||||
),
|
||||
usage.input_cache_write.to_string(),
|
||||
);
|
||||
persisted_data.insert(
|
||||
format!(
|
||||
"{}{}.cost_in_cents",
|
||||
RUNTIME_TAG_TOKEN_USAGE_PREFIX, usage.model_id
|
||||
),
|
||||
usage.cost_in_cents.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Write to debug file for backward compatibility
|
||||
write_to_debug_file(&format!(
|
||||
"{TOTAL_REQUEST_COST_PREFIX}{total_request_cost}"
|
||||
));
|
||||
|
||||
write_to_debug_file(&format!("{TOTAL_EXCHANGES_PREFIX}{total_exchanges}"));
|
||||
|
||||
write_to_debug_file(&format!(
|
||||
"{TOTAL_TOKEN_USAGE_PREFIX}{}",
|
||||
token_usage
|
||||
.iter()
|
||||
.map(|usage| format!(
|
||||
"model_id={},total_input={},output={},input_cache_read={},input_cache_write={},cost_in_cents={}",
|
||||
usage.model_id,
|
||||
usage.total_input,
|
||||
usage.output,
|
||||
usage.input_cache_read,
|
||||
usage.input_cache_write,
|
||||
usage.cost_in_cents
|
||||
))
|
||||
.collect::<Vec<_>>()
|
||||
.join("|")
|
||||
));
|
||||
|
||||
write_to_debug_file("\nConversation Exchanges:\n");
|
||||
for (i, exchange) in conversation.all_exchanges().into_iter().enumerate() {
|
||||
// Add timestamp for each exchange
|
||||
let exchange_time = chrono::DateTime::<chrono::Utc>::from(exchange.start_time)
|
||||
.format("%Y-%m-%dT%H:%M:%S%.3fZ");
|
||||
write_to_debug_file(&format!(
|
||||
"\n--- Exchange {} (Started: {}) ---",
|
||||
i + 1,
|
||||
exchange_time
|
||||
));
|
||||
|
||||
for input in &exchange.input {
|
||||
write_to_debug_file(&format!("\nInput:\n\n{input}\n"));
|
||||
}
|
||||
write_to_debug_file(&format!("Output:\n{}", &exchange.output_status))
|
||||
}
|
||||
|
||||
// Add completion timestamp
|
||||
let completion_time = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ");
|
||||
write_to_debug_file(&format!(
|
||||
"\n========Debug Info Complete ({completion_time})=========="
|
||||
));
|
||||
})
|
||||
}
|
||||
|
||||
// Get debug output file path from environment
|
||||
pub fn open_debug_file_from_env(env_var: &str) -> Option<File> {
|
||||
let file_path = std::env::var(env_var).ok();
|
||||
if let Some(file_path) = &file_path {
|
||||
// Clear the file if it exists
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
Some(
|
||||
std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(file_path)
|
||||
.expect("Failed to open debug output file"),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
use std::{fs::read, io::Cursor, path::Path, time::Duration};
|
||||
|
||||
use prost::Message;
|
||||
use warpui::{async_assert, integration::TestStep, SingletonEntity};
|
||||
|
||||
use crate::ai::execution_profiles::profiles::AIExecutionProfilesModel;
|
||||
use crate::ai::execution_profiles::ActionPermission;
|
||||
use crate::ai::llms::{LLMId, LLMPreferences};
|
||||
use crate::integration_testing::agent_mode::ConversationTarget;
|
||||
use crate::integration_testing::{
|
||||
agent_mode::{assert_latest_task_succeeds_or_blocked, assert_task_is_blocked},
|
||||
step::{new_step_with_default_assertions, new_step_with_default_assertions_for_pane},
|
||||
terminal::assert_input_is_focused,
|
||||
view_getters::terminal_view,
|
||||
};
|
||||
|
||||
pub const AGENT_MODE_RUNNING_STEP_GROUP_NAME: &str = "Agent mode running";
|
||||
|
||||
use super::hydrate_ai_conversation_assertion;
|
||||
|
||||
/// Assumes that the terminal input is currently not in AI input mode.
|
||||
pub fn enter_agent_view() -> TestStep {
|
||||
new_step_with_default_assertions("Enter Agent View")
|
||||
.with_keystrokes(&["ctrl-shift-enter"])
|
||||
.add_named_assertion(
|
||||
"Assert that we are in Agent View and AI input mode",
|
||||
move |app, window_id| {
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
terminal_view.read(app, |terminal_view, app| {
|
||||
let is_ai_input_mode = terminal_view
|
||||
.input()
|
||||
.read(app, |input, app| input.input_type(app).is_ai());
|
||||
let agent_view_state = {
|
||||
let model = terminal_view.model.lock();
|
||||
model.block_list().agent_view_state().clone()
|
||||
};
|
||||
async_assert!(
|
||||
is_ai_input_mode && agent_view_state.is_active(),
|
||||
"Expected fullscreen Agent View + AI input mode, got agent_view_state={agent_view_state:?}, is_ai_input_mode={is_ai_input_mode}"
|
||||
)
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Assumes that the terminal input is currently in AI input mode.
|
||||
pub fn exit_agent_view() -> TestStep {
|
||||
new_step_with_default_assertions("Exit Agent View")
|
||||
.with_keystrokes(&["escape"])
|
||||
.add_named_assertion(
|
||||
"Assert that we exited Agent View and are not in AI input mode",
|
||||
move |app, window_id| {
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
terminal_view.read(app, |terminal_view, app| {
|
||||
let is_ai_input_mode = terminal_view
|
||||
.input()
|
||||
.read(app, |input, app| input.input_type(app).is_ai());
|
||||
let agent_view_state = {
|
||||
let model = terminal_view.model.lock();
|
||||
model.block_list().agent_view_state().clone()
|
||||
};
|
||||
async_assert!(
|
||||
!is_ai_input_mode && !agent_view_state.is_active(),
|
||||
"Expected inactive Agent View + non-AI input mode, got agent_view_state={agent_view_state:?}, is_ai_input_mode={is_ai_input_mode}"
|
||||
)
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Hydrates a conversation from a protobuf file.
|
||||
/// File should be generated into the `input_data` directory.
|
||||
/// See the agent_mode_eval README for more details.
|
||||
pub fn hydrate_ai_conversation(file_name: &str) -> TestStep {
|
||||
let file_bytes = get_input_data(file_name);
|
||||
let Ok(request) = warp_multi_agent_api::Request::decode(file_bytes) else {
|
||||
panic!("Failed to decode request from protobuf");
|
||||
};
|
||||
|
||||
let tasks = request
|
||||
.task_context
|
||||
.map(|ctx| ctx.tasks)
|
||||
.unwrap_or_default();
|
||||
|
||||
new_step_with_default_assertions("Hydrate AI conversation").add_named_assertion(
|
||||
"Assert that conversation was hydrated successfully",
|
||||
hydrate_ai_conversation_assertion(tasks),
|
||||
)
|
||||
}
|
||||
|
||||
/// Attach the latest block in the blocklist (command + output) to the AI query.
|
||||
pub fn attach_recent_block_as_context() -> TestStep {
|
||||
TestStep::new("Attach last block as context").add_named_assertion(
|
||||
"Attach last block as context",
|
||||
|app, window_id| {
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
terminal_view.update(app, |view, ctx| {
|
||||
let last_index = {
|
||||
let model = view.model.lock();
|
||||
model.block_list().last_non_hidden_block_by_index()
|
||||
};
|
||||
if let Some(idx) = last_index {
|
||||
view.integration_test_change_block_selection_to_single(idx, ctx);
|
||||
}
|
||||
});
|
||||
|
||||
terminal_view.read(app, |view, ctx| {
|
||||
let count = view
|
||||
.ai_context_model()
|
||||
.as_ref(ctx)
|
||||
.pending_context_block_ids()
|
||||
.len();
|
||||
async_assert!(
|
||||
count == 1,
|
||||
"Expected exactly 1 attached context block, got {count}"
|
||||
)
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// This will fail immediately on any error responses.
|
||||
pub fn submit_ai_query_and_wait_until_done(query: &str, timeout: Duration) -> TestStep {
|
||||
submit_ai_query(query, timeout)
|
||||
.add_named_assertion(
|
||||
"Assert the agent task is complete",
|
||||
assert_latest_task_succeeds_or_blocked(ConversationTarget::Active, None),
|
||||
)
|
||||
.add_named_assertion(
|
||||
"Assert that that input has been returned to the user",
|
||||
assert_input_is_focused(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Submits an AI query and waits until the task is blocked (waiting for user approval).
|
||||
/// This is useful for tests where auto-execution is disabled and you want to verify
|
||||
/// the command that would be executed without actually running it.
|
||||
pub fn submit_ai_query_and_wait_until_blocked(query: &str, timeout: Duration) -> TestStep {
|
||||
submit_ai_query(query, timeout).add_named_assertion(
|
||||
"Assert the agent task is blocked",
|
||||
assert_task_is_blocked(ConversationTarget::Active),
|
||||
)
|
||||
}
|
||||
|
||||
// Runs an AI query without waiting for anything.
|
||||
// This is useful if you expect a specific sequence of responses (e.g. expect a certain command to be requested immediately),
|
||||
// since it lets you make assertions on responses as they become ready and fail early instead of waiting for the agent to finish all its turns.
|
||||
pub fn submit_ai_query(query: &str, timeout: Duration) -> TestStep {
|
||||
new_step_with_default_assertions_for_pane(&format!("Enter AI query: {query}"), 0, 0)
|
||||
.set_timeout(timeout)
|
||||
.set_step_group_name(AGENT_MODE_RUNNING_STEP_GROUP_NAME)
|
||||
.with_typed_characters(&[query])
|
||||
.with_keystrokes(&["enter"])
|
||||
.add_named_assertion(
|
||||
"Print conversation ID to stdout",
|
||||
print_conversation_id_assertion(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns an assertion that prints the conversation ID to stdout once available.
|
||||
/// This assertion will poll until the conversation token is received from the server.
|
||||
fn print_conversation_id_assertion(
|
||||
) -> impl FnMut(&mut warpui::App, warpui::WindowId) -> warpui::integration::AssertionOutcome {
|
||||
|app, window_id| {
|
||||
use crate::BlocklistAIHistoryModel;
|
||||
use warpui::integration::AssertionOutcome;
|
||||
let terminal_view = terminal_view(app, window_id, 0, 0);
|
||||
BlocklistAIHistoryModel::handle(app).read(app, |history_model, _| {
|
||||
if let Some(conversation) = history_model.active_conversation(terminal_view.id()) {
|
||||
if let Some(token) = conversation.server_conversation_token() {
|
||||
// The debug link within the container will be using host.docker.internal, but we're opening
|
||||
// from outside the container.
|
||||
let debug_link = token
|
||||
.debug_link()
|
||||
.replace("host.docker.internal", "localhost");
|
||||
println!("Conversation ID (debug link): {debug_link}");
|
||||
return AssertionOutcome::Success;
|
||||
}
|
||||
}
|
||||
// If we don't have a conversation token yet, keep polling
|
||||
AssertionOutcome::failure("Waiting for conversation token to be available".to_string())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the preferred agent mode LLM. This is the base model for agent and inline AI conversations.
|
||||
pub fn set_preferred_agent_mode_llm(llm_id: &str) -> TestStep {
|
||||
let llm_id = LLMId::from(llm_id);
|
||||
TestStep::new(&format!("Set preferred agent mode LLM to {llm_id}")).add_named_assertion(
|
||||
"Update preferred agent mode LLM",
|
||||
move |app, window_id| {
|
||||
let llm_id = llm_id.clone();
|
||||
let terminal_view_id = terminal_view(app, window_id, 0, 0).id();
|
||||
LLMPreferences::handle(app).update(app, |llm_preferences, ctx| {
|
||||
// Validate that the LLM ID is actually available. We only do this
|
||||
// for the base model, since the coding and planning models are
|
||||
// currently unused in the product.
|
||||
assert!(
|
||||
llm_preferences.is_available_agent_mode_llm(&llm_id),
|
||||
"LLM ID '{llm_id}' is not a valid agent mode LLM",
|
||||
);
|
||||
llm_preferences.update_preferred_agent_mode_llm(&llm_id, terminal_view_id, ctx);
|
||||
});
|
||||
async_assert!(true, "Successfully updated preferred agent mode LLM")
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Sets the preferred coding LLM. Note that the server currently ignores this.
|
||||
pub fn set_preferred_coding_llm(llm_id: &str) -> TestStep {
|
||||
let llm_id = LLMId::from(llm_id);
|
||||
TestStep::new(&format!("Set preferred coding LLM to {llm_id}")).add_named_assertion(
|
||||
"Update preferred coding LLM",
|
||||
move |app, window_id| {
|
||||
let llm_id = llm_id.clone();
|
||||
let terminal_view_id = terminal_view(app, window_id, 0, 0).id();
|
||||
LLMPreferences::handle(app).update(app, |llm_preferences, ctx| {
|
||||
llm_preferences.update_preferred_coding_llm(&llm_id, Some(terminal_view_id), ctx);
|
||||
});
|
||||
async_assert!(true, "Successfully updated preferred coding LLM")
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn get_input_data(file_name: &str) -> Cursor<Vec<u8>> {
|
||||
let input_data_dir = std::env::var("INPUT_DATA_DIR").expect(
|
||||
"INPUT_DATA_DIR is not set. This is needed to hydrate conversations from eval tests.",
|
||||
);
|
||||
let path = Path::new(&input_data_dir).join(file_name);
|
||||
Cursor::new(read(&path).expect("Failed to read binary input data"))
|
||||
}
|
||||
|
||||
/// Sets the execution profile to not auto-execute commands.
|
||||
/// This changes the `execute_commands` permission from `AlwaysAllow` to `AlwaysAsk`,
|
||||
/// which means commands will be proposed but not automatically executed.
|
||||
pub fn set_execution_profile_no_auto_execute() -> TestStep {
|
||||
TestStep::new("Set execution profile to not auto-execute commands").add_named_assertion(
|
||||
"Update execution profile",
|
||||
|app, _window_id| {
|
||||
AIExecutionProfilesModel::handle(app).update(app, |profiles, ctx| {
|
||||
let default_profile_id = *profiles.default_profile(ctx).id();
|
||||
profiles.set_execute_commands(
|
||||
default_profile_id,
|
||||
&ActionPermission::AlwaysAsk,
|
||||
ctx,
|
||||
);
|
||||
});
|
||||
async_assert!(true, "Successfully updated execution profile")
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
// User default keys
|
||||
const IS_ACTIVE_AI_ENABLED: &str = "IsActiveAIEnabled";
|
||||
const INTELLIGENT_AUTOSUGGESTIONS_ENABLED: &str = "IntelligentAutosuggestionsEnabled";
|
||||
const NATURAL_LANGUAGE_AUTOSUGGESTIONS_ENABLED: &str = "NaturalLanguageAutosuggestionsEnabled";
|
||||
const AGENT_MODE_QUERY_SUGGESTIONS_ENABLED: &str = "AgentModeQuerySuggestionsEnabled";
|
||||
const CODE_SUGGESTIONS_ENABLED: &str = "CodeSuggestionsEnabled";
|
||||
|
||||
pub fn user_defaults_map_with_active_ai(enabled: bool) -> HashMap<String, String> {
|
||||
HashMap::from_iter([
|
||||
(
|
||||
INTELLIGENT_AUTOSUGGESTIONS_ENABLED.to_owned(),
|
||||
enabled.to_string(),
|
||||
),
|
||||
(
|
||||
AGENT_MODE_QUERY_SUGGESTIONS_ENABLED.to_owned(),
|
||||
enabled.to_string(),
|
||||
),
|
||||
(CODE_SUGGESTIONS_ENABLED.to_owned(), enabled.to_string()),
|
||||
(
|
||||
NATURAL_LANGUAGE_AUTOSUGGESTIONS_ENABLED.to_owned(),
|
||||
enabled.to_string(),
|
||||
),
|
||||
(IS_ACTIVE_AI_ENABLED.to_owned(), enabled.to_string()),
|
||||
])
|
||||
}
|
||||
|
||||
/// User defaults for predictable AI input behavior needed in evals.
|
||||
///
|
||||
/// This allows tests to more reliably enter and exit AI input mode.
|
||||
///
|
||||
/// * UDI is enabled
|
||||
/// * Natural language detection is disabled
|
||||
pub fn user_defaults_map_for_ai_input() -> HashMap<String, String> {
|
||||
HashMap::from_iter([
|
||||
(
|
||||
"AIAutoDetectionEnabled".to_owned(),
|
||||
serde_json::to_string(&false).unwrap(),
|
||||
),
|
||||
(
|
||||
"InputBoxTypeSetting".to_owned(),
|
||||
serde_json::to_string("Universal").unwrap(),
|
||||
),
|
||||
])
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/// Check that a CLI command contains a sequence of tokens in order.
|
||||
/// Useful to ignore flags that may be present between tokens.
|
||||
pub fn command_contains_sequence(cmd: &str, sequence: &[&str]) -> bool {
|
||||
let mut required_index = 0;
|
||||
|
||||
for token in cmd.split_whitespace() {
|
||||
if token == sequence[required_index] {
|
||||
required_index += 1;
|
||||
if required_index == sequence.len() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
pub fn is_running_in_docker() -> bool {
|
||||
std::path::Path::new("/.dockerenv").exists()
|
||||
}
|
||||
|
||||
pub fn get_base_server_url() -> String {
|
||||
if is_running_in_docker() {
|
||||
"http://host.docker.internal:8080".to_string()
|
||||
} else {
|
||||
"http://localhost:8080".to_string()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user