first pass of merging in warp (doesn't build)

This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
+61 -43
View File
@@ -1,26 +1,21 @@
use std::{path::PathBuf, time::Duration};
use std::path::PathBuf;
use std::time::Duration;
use galaxy_core::features::FeatureFlag;
use itertools::Itertools as _;
use uuid::Uuid;
use warp_multi_agent_api as api;
use crate::{
agent::{
action::{
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, ReadSkillRequest,
SearchCodebaseRequest, ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest,
UseComputerRequest,
},
action_result::{AnyFileContent, FileContext},
convert::ToolToAIAgentActionError,
FileLocations,
},
diff_validation::{ParsedDiff, V4AHunk},
document::AIDocumentId,
skills::SkillReference,
use crate::agent::action::{
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, SearchCodebaseRequest,
ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest, UseComputerRequest,
};
use crate::agent::action_result::{AnyFileContent, FileContext};
use crate::agent::convert::ToolToAIAgentActionError;
use crate::agent::FileLocations;
use crate::diff_validation::{ParsedDiff, V4AHunk};
use crate::document::AIDocumentId;
impl From<api::message::tool_call::RunShellCommand> for AIAgentActionType {
fn from(value: api::message::tool_call::RunShellCommand) -> Self {
@@ -430,10 +425,11 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
.actions
.into_iter()
.map(|action| {
let target = convert_computer_use_target(action.target);
let Some(action_type) = action.r#type else {
return Err(ToolToAIAgentActionError::MissingComputerUseActionType);
};
match action_type {
let action = match action_type {
use_computer::action::Type::MouseMove(mouse_move) => {
Ok(computer_use::Action::MouseMove {
to: coordinates_to_vec(mouse_move.to.as_ref())?,
@@ -481,7 +477,9 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
let key = convert_key(key_up.key)?;
Ok(computer_use::Action::KeyUp { key })
}
}
}?;
log_window_target_coord(&action, target);
Ok(computer_use::TargetedAction { action, target })
})
.try_collect()?;
let screenshot_params = value
@@ -505,19 +503,6 @@ impl From<api::message::tool_call::RequestComputerUse> for AIAgentActionType {
}
}
impl TryFrom<api::message::tool_call::ReadSkill> for AIAgentActionType {
type Error = ToolToAIAgentActionError;
fn try_from(value: api::message::tool_call::ReadSkill) -> Result<Self, Self::Error> {
match value.skill_reference {
Some(reference) => Ok(AIAgentActionType::ReadSkill(ReadSkillRequest {
skill: SkillReference::from(reference),
})),
None => Err(ToolToAIAgentActionError::MissingSkillReference),
}
}
}
impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
fn from(value: api::message::tool_call::FetchConversation) -> Self {
AIAgentActionType::FetchConversation {
@@ -526,18 +511,6 @@ impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
}
}
impl From<api::message::tool_call::read_skill::SkillReference> for SkillReference {
fn from(value: api::message::tool_call::read_skill::SkillReference) -> Self {
use warp_multi_agent_api::message::tool_call::read_skill::SkillReference as ApiSkillReference;
match value {
ApiSkillReference::SkillPath(skill_path) => {
SkillReference::Path(PathBuf::from(skill_path))
}
ApiSkillReference::BundledSkillId(id) => SkillReference::BundledSkillId(id),
}
}
}
/// Converts API ScreenshotParams to the internal computer_use type.
fn convert_screenshot_params(
params: api::message::tool_call::ScreenshotParams,
@@ -556,6 +529,51 @@ fn convert_screenshot_params(
max_long_edge_px: (params.max_long_edge_px > 0).then_some(params.max_long_edge_px as usize),
max_total_px: (params.max_total_px > 0).then_some(params.max_total_px as usize),
region,
target: convert_computer_use_target(params.target),
}
}
/// Converts an optional API `ComputerUseTarget` into the internal computer_use target. An absent
/// or `Screen` target maps to the legacy whole-screen behavior.
fn convert_computer_use_target(
target: Option<api::message::tool_call::ComputerUseTarget>,
) -> computer_use::Target {
use api::message::tool_call::computer_use_target::Target as ApiTarget;
match target.and_then(|t| t.target) {
// The proto window id is an opaque string; on macOS it is a CGWindowID, so parse it back
// to a u32. An unparseable id is treated as no valid window target and falls back to the
// legacy whole-screen behavior rather than panicking.
Some(ApiTarget::Window(window)) => match window.window_id.parse::<u32>() {
Ok(window_id) => computer_use::Target::Window {
window_id,
pid: window.pid,
},
Err(_) => computer_use::Target::Screen,
},
Some(ApiTarget::Screen(_)) | None => computer_use::Target::Screen,
}
}
/// Logs the raw server-provided coordinates for a window-targeted computer-use action, so the
/// agent-driven coordinate conversion can be compared against where the click actually lands.
/// Gated on COMPUTER_USE_DEBUG and routed through `log` so it surfaces in the app's log file.
fn log_window_target_coord(action: &computer_use::Action, target: computer_use::Target) {
if std::env::var_os("COMPUTER_USE_DEBUG").is_none() {
return;
}
let computer_use::Target::Window { window_id, pid } = target else {
return;
};
let coord = match action {
computer_use::Action::MouseMove { to } => Some(("mouse_move", to.x(), to.y())),
computer_use::Action::MouseDown { at, .. } => Some(("mouse_down", at.x(), at.y())),
computer_use::Action::MouseWheel { at, .. } => Some(("mouse_wheel", at.x(), at.y())),
_ => None,
};
if let Some((kind, x, y)) = coord {
log::info!(
"[computer_use] server->client {kind} window#={window_id} pid={pid} raw_coord=({x},{y})"
);
}
}
+129 -25
View File
@@ -1,34 +1,32 @@
mod convert;
use std::{fmt::Display, ops::Range, path::PathBuf, time::Duration};
use std::fmt::Display;
use std::ops::Range;
use std::path::PathBuf;
use std::time::Duration;
use galaxy_terminal::model::BlockId;
use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use strum_macros::EnumDiscriminants;
use uuid::Uuid;
use crate::{
agent::{
action_result::{
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult,
CreateDocumentsResult, EditDocumentsResult, FetchConversationResult, FileGlobResult,
FileGlobV2Result, GrepResult, InsertReviewCommentsResult, ReadDocumentsResult,
ReadFilesResult, ReadMCPResourceResult, ReadShellCommandOutputResult, ReadSkillResult,
RequestCommandOutputResult, RequestComputerUseResult, RequestFileEditsResult,
SearchCodebaseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
SuggestNewConversationResult, SuggestPromptResult,
TransferShellCommandControlToUserResult, UploadArtifactResult, UseComputerResult,
WriteToLongRunningShellCommandResult,
},
AIAgentCitation, FileLocations,
},
diff_validation::ParsedDiff,
document::AIDocumentId,
skills::SkillReference,
};
pub use warp_multi_agent_api::LifecycleEventType;
use crate::agent::action_result::{
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult, CreateDocumentsResult,
EditDocumentsResult, FetchConversationResult, FileGlobResult, FileGlobV2Result, GrepResult,
InsertReviewCommentsResult, ReadDocumentsResult, ReadFilesResult, ReadMCPResourceResult,
ReadShellCommandOutputResult, ReadSkillResult, RequestCommandOutputResult,
RequestComputerUseResult, RequestFileEditsResult, RunAgentsResult, SearchCodebaseResult,
SendMessageToAgentResult, StartAgentResult, StartAgentVersion, SuggestNewConversationResult,
SuggestPromptResult, TransferShellCommandControlToUserResult, UploadArtifactResult,
UseComputerResult, WaitForEventsResult, WriteToLongRunningShellCommandResult,
};
use crate::agent::{AIAgentCitation, FileLocations};
use crate::diff_validation::ParsedDiff;
use crate::document::AIDocumentId;
use crate::skills::SkillReference;
#[derive(Debug, Clone, Eq, PartialEq, EnumDiscriminants)]
pub enum AIAgentActionType {
/// The AI requested the output for a given command to be retrieved as context in responding to
@@ -48,7 +46,7 @@ pub enum AIAgentActionType {
/// result instead.
wait_until_completion: bool,
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke pager.
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke a pager.
uses_pager: Option<bool>,
/// The AI's rationale for requesting a command.
@@ -144,7 +142,9 @@ pub enum AIAgentActionType {
FetchConversation {
conversation_id: String,
},
// TODO(QUALITY-788): Delete legacy start_agent/start_agent_v2 action support once
// old preview orchestration history no longer needs parse/display/result compatibility.
// Linear issue: QUALITY-788.
StartAgent {
version: StartAgentVersion,
name: String,
@@ -167,6 +167,68 @@ pub enum AIAgentActionType {
AskUserQuestion {
questions: Vec<AskUserQuestionItem>,
},
/// AI requested batched orchestration of one-or-more child agents that
/// share run-wide configuration (model, harness, execution mode).
/// The full per-child prompt is computed at dispatch time as
/// `base_prompt + "\n\n" + agent_run_configs[i].prompt` (or just
/// `base_prompt` when the per-agent `prompt` is empty).
RunAgents(RunAgentsRequest),
/// Synthesized from a server-emitted Message::ToolCall::WaitForEvents;
/// dispatched by WaitForEventsExecutor.
WaitForEvents {
/// tool_call_id of the unresolved WaitForEvents call; used to
/// match inbound resume signals.
tool_call_id: String,
/// 0 means "unset" (prost flat-scalar convention); the executor
/// falls back to a default.
idle_timeout_seconds: i32,
},
}
/// Run-wide + per-agent configuration for a `RunAgents` tool call.
///
/// Mirrors the proto `RunAgents` message. Server-resolved fields
/// (`model_id`, `harness_type`, `execution_mode`'s remote details) are
/// folded in by the server's final tool-call re-emission once the
/// payload is complete; the client renders the full layout from a
/// fully-resolved instance only.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct RunAgentsRequest {
pub summary: String,
pub base_prompt: String,
pub skills: Vec<SkillReference>,
pub model_id: String,
pub harness_type: String,
pub execution_mode: RunAgentsExecutionMode,
pub agent_run_configs: Vec<RunAgentsAgentRunConfig>,
pub plan_id: String,
/// Resolved client-side at dispatch time; not serialized to the wire.
pub harness_auth_secret_name: Option<String>,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum RunAgentsExecutionMode {
Local,
Remote {
environment_id: String,
worker_host: String,
computer_use_enabled: bool,
},
}
impl RunAgentsExecutionMode {
pub fn is_remote(&self) -> bool {
matches!(self, Self::Remote { .. })
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct RunAgentsAgentRunConfig {
pub name: String,
pub prompt: String,
pub title: String,
}
#[derive(Debug, Clone, Eq, PartialEq)]
@@ -175,6 +237,11 @@ pub enum StartAgentExecutionMode {
/// `None` selects the legacy embedded local child-agent flow.
/// `Some(...)` selects a third-party CLI harness to launch locally.
harness_type: Option<String>,
/// `None` inherits the parent agent's preferred LLM (legacy behavior).
/// `Some(_)` overrides the child's preferred LLM with the supplied
/// model id (used by the orchestrate confirmation card so the user's
/// model selection is honored on local launches).
model_id: Option<String>,
},
Remote {
environment_id: String,
@@ -184,18 +251,27 @@ pub enum StartAgentExecutionMode {
worker_host: String,
harness_type: String,
title: String,
/// Name of a managed secret to forward as the authentication
/// credential for the remote child when running a non-Oz harness.
/// `None` means no client-side secret was selected — the remote
/// environment falls back to its own ambient credentials.
auth_secret_name: Option<String>,
},
}
impl StartAgentExecutionMode {
/// Constructs a local execution mode using the legacy v1 default harness.
pub fn local_with_defaults() -> Self {
Self::Local { harness_type: None }
Self::Local {
harness_type: None,
model_id: None,
}
}
/// Constructs a local execution mode for a specific third-party harness.
pub fn local_harness(harness_type: String) -> Self {
Self::Local {
harness_type: Some(harness_type),
model_id: None,
}
}
/// Constructs a remote execution mode using the legacy v1 defaults for
@@ -209,6 +285,7 @@ impl StartAgentExecutionMode {
worker_host: String::new(),
harness_type: String::new(),
title: String::new(),
auth_secret_name: None,
}
}
}
@@ -317,6 +394,10 @@ impl AIAgentActionType {
Self::AskUserQuestion { .. } => {
AIAgentActionResultType::AskUserQuestion(AskUserQuestionResult::Cancelled)
}
Self::RunAgents(_) => AIAgentActionResultType::RunAgents(RunAgentsResult::Cancelled),
Self::WaitForEvents { .. } => {
AIAgentActionResultType::WaitForEvents(WaitForEventsResult::Cancelled)
}
}
}
@@ -362,6 +443,10 @@ impl AIAgentActionType {
Self::AskUserQuestion { questions } => {
format!("Ask user {} question(s)", questions.len())
}
Self::RunAgents(req) => {
format!("Orchestrate {} agent(s)", req.agent_run_configs.len())
}
Self::WaitForEvents { .. } => "Wait for events".to_string(),
}
}
}
@@ -534,6 +619,24 @@ impl Display for AIAgentActionType {
AIAgentActionType::AskUserQuestion { questions } => {
write!(f, "AskUserQuestion: {} question(s)", questions.len())
}
AIAgentActionType::RunAgents(req) => {
let names = req
.agent_run_configs
.iter()
.map(|c| c.name.as_str())
.collect::<Vec<_>>()
.join(", ");
write!(f, "Orchestrate: summary='{}' agents=[{names}]", req.summary,)
}
AIAgentActionType::WaitForEvents {
tool_call_id,
idle_timeout_seconds,
} => {
write!(
f,
"WaitForEvents: tool_call_id={tool_call_id} idle_timeout_seconds={idle_timeout_seconds}"
)
}
}
}
}
@@ -665,7 +768,8 @@ pub struct CreateDocumentsRequest {
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct UseComputerRequest {
pub action_summary: String,
pub actions: Vec<computer_use::Action>,
/// Each action carries the surface (screen or a specific window) it targets.
pub actions: Vec<computer_use::TargetedAction>,
/// If set, a screenshot will be captured after the actions are executed.
pub screenshot_params: Option<computer_use::ScreenshotParams>,
}
+191 -9
View File
@@ -1,12 +1,20 @@
use warp_multi_agent_api::{
self as api,
apply_file_diffs_result::success::UpdatedFileContent,
ask_user_question_result::answer_item::{self, Answer as AskUserQuestionAnswer},
use chrono::{DateTime, Local};
use warp_multi_agent_api::apply_file_diffs_result::success::UpdatedFileContent;
use warp_multi_agent_api::ask_user_question_result::answer_item::{
self, Answer as AskUserQuestionAnswer,
};
use crate::agent::{action_result::ShellCommandError, convert::ConvertToAPITypeError};
use warp_multi_agent_api::{self as api};
use super::*;
use crate::agent::action_result::ShellCommandError;
use crate::agent::convert::ConvertToAPITypeError;
fn local_datetime_to_timestamp(timestamp: DateTime<Local>) -> prost_types::Timestamp {
prost_types::Timestamp {
seconds: timestamp.timestamp(),
nanos: timestamp.timestamp_subsec_nanos() as i32,
}
}
impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_result::Result {
type Error = ConvertToAPITypeError;
@@ -18,7 +26,8 @@ impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_resu
block_id,
output,
exit_code,
..
start_ts,
completed_ts,
} => Ok(
api::request::input::tool_call_result::Result::RunShellCommand(
#[allow(deprecated)]
@@ -31,6 +40,8 @@ impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_resu
command_id: block_id.to_string(),
output,
exit_code: exit_code.value(),
start_ts: start_ts.map(local_datetime_to_timestamp),
finish_ts: completed_ts.map(local_datetime_to_timestamp),
},
)),
},
@@ -127,7 +138,7 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
},
),
),
WriteToLongRunningShellCommandResult::CommandFinished { block_id, output, exit_code, .. } => Ok(
WriteToLongRunningShellCommandResult::CommandFinished { block_id, output, exit_code, start_ts, completed_ts } => Ok(
api::request::input::tool_call_result::Result::WriteToLongRunningShellCommand(
api::WriteToLongRunningShellCommandResult {
result: Some(api::write_to_long_running_shell_command_result::Result::CommandFinished(
@@ -135,6 +146,8 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
command_id: block_id.to_string(),
output,
exit_code: exit_code.value(),
start_ts: start_ts.map(local_datetime_to_timestamp),
finish_ts: completed_ts.map(local_datetime_to_timestamp),
}
))
},
@@ -797,7 +810,8 @@ impl TryFrom<ReadShellCommandOutputResult> for api::request::input::tool_call_re
block_id,
output,
exit_code,
..
start_ts,
completed_ts,
} => Ok(
api::request::input::tool_call_result::Result::ReadShellCommandOutput(
api::ReadShellCommandOutputResult {
@@ -807,6 +821,8 @@ impl TryFrom<ReadShellCommandOutputResult> for api::request::input::tool_call_re
command_id: block_id.to_string(),
output,
exit_code: exit_code.value(),
start_ts: start_ts.map(local_datetime_to_timestamp),
finish_ts: completed_ts.map(local_datetime_to_timestamp),
},
)),
},
@@ -891,6 +907,8 @@ impl TryFrom<TransferShellCommandControlToUserResult>
block_id,
output,
exit_code,
start_ts,
completed_ts,
} => Ok(
api::request::input::tool_call_result::Result::TransferShellCommandControlToUser(
api::TransferShellCommandControlToUserResult {
@@ -900,6 +918,8 @@ impl TryFrom<TransferShellCommandControlToUserResult>
command_id: block_id.to_string(),
output,
exit_code: exit_code.value(),
start_ts: start_ts.map(local_datetime_to_timestamp),
finish_ts: completed_ts.map(local_datetime_to_timestamp),
},
),
),
@@ -1109,6 +1129,7 @@ impl TryFrom<RequestComputerUseResult> for api::request::input::tool_call_result
RequestComputerUseResult::Approved {
screenshot,
platform,
windows,
} => Ok(
api::request::input::tool_call_result::Result::RequestComputerUse(
api::RequestComputerUseResult {
@@ -1125,6 +1146,7 @@ impl TryFrom<RequestComputerUseResult> for api::request::input::tool_call_result
height: screenshot.height as i32,
}),
platform: convert_platform(platform).into(),
windows: windows.into_iter().map(convert_window_info).collect(),
},
)),
},
@@ -1158,6 +1180,9 @@ impl TryFrom<UseComputerResult> for api::request::input::tool_call_result::Resul
fn try_from(result: UseComputerResult) -> Result<Self, Self::Error> {
match result {
UseComputerResult::Success(result) => {
// Copy out the captured-window metadata (if any) before the owned fields of
// `result` are moved into the message below.
let captured = result.captured_window;
Ok(api::request::input::tool_call_result::Result::UseComputer(
api::UseComputerResult {
result: Some(api::use_computer_result::Result::Success(
@@ -1169,6 +1194,20 @@ impl TryFrom<UseComputerResult> for api::request::input::tool_call_result::Resul
height: s.height as i32,
}),
cursor_position: result.cursor_position.map(vec_to_coordinates),
windows: result
.windows
.into_iter()
.map(convert_window_info)
.collect(),
// The window id is an opaque string on the wire; on macOS it is a
// CGWindowID, so format the u32 back to a string at the boundary.
captured_window: captured.map(|c| {
api::use_computer_result::success::CapturedWindow {
window_id: c.window_id.to_string(),
width_px: c.width_px,
height_px: c.height_px,
}
}),
},
)),
},
@@ -1205,6 +1244,18 @@ fn vec_to_coordinates(vec: computer_use::Vector2I) -> api::Coordinates {
}
}
/// Converts a computer_use window record into the API `WindowInfo` message.
fn convert_window_info(window: computer_use::WindowInfo) -> api::WindowInfo {
api::WindowInfo {
// The window id travels as an opaque string; on macOS it is a CGWindowID (u32).
window_id: window.window_id.to_string(),
pid: window.pid,
app_name: window.app_name,
title: window.title,
layer: window.layer,
}
}
fn convert_platform(
platform: computer_use::Platform,
) -> api::request_computer_use_result::approved::Platform {
@@ -1463,6 +1514,121 @@ impl From<AskUserQuestionResult> for api::request::input::tool_call_result::Resu
}
}
impl From<RunAgentsLaunchedExecutionMode>
for api::run_agents_result::launched::ResolvedExecutionMode
{
fn from(mode: RunAgentsLaunchedExecutionMode) -> Self {
match mode {
RunAgentsLaunchedExecutionMode::Local => {
api::run_agents_result::launched::ResolvedExecutionMode::Local(
api::run_agents::Local {},
)
}
RunAgentsLaunchedExecutionMode::Remote {
environment_id,
worker_host,
computer_use_enabled,
} => api::run_agents_result::launched::ResolvedExecutionMode::Remote(
api::run_agents::Remote {
environment_id,
worker_host,
computer_use_enabled,
},
),
}
}
}
impl From<RunAgentsAgentOutcome> for api::run_agents_result::AgentOutcome {
fn from(outcome: RunAgentsAgentOutcome) -> Self {
let result = match outcome.kind {
RunAgentsAgentOutcomeKind::Launched { agent_id } => {
api::run_agents_result::agent_outcome::Result::Launched(
api::run_agents_result::LaunchedAgent { agent_id },
)
}
RunAgentsAgentOutcomeKind::Failed { error } => {
api::run_agents_result::agent_outcome::Result::Failed(
api::run_agents_result::FailedAgent { error },
)
}
};
api::run_agents_result::AgentOutcome {
name: outcome.name,
result: Some(result),
}
}
}
/// Maps a client-side harness string identifier (e.g. "oz", "claude")
/// to the new proto `Harness` oneof. Returns `None` for empty,
/// unrecognized, or `"unknown"` strings; callers leave
/// `resolved_harness` unset in that case.
pub(super) fn build_api_harness(harness_type: &str) -> Option<api::Harness> {
let normalized = harness_type.trim().to_ascii_lowercase().replace('_', "-");
let variant = match normalized.as_str() {
"oz" => api::harness::Variant::Oz(api::harness::Oz {}),
"claude" | "claude-code" => api::harness::Variant::ClaudeCode(api::harness::ClaudeCode {}),
"opencode" | "open-code" => api::harness::Variant::OpenCode(api::harness::OpenCode {}),
"gemini" => api::harness::Variant::Gemini(api::harness::Gemini {}),
"codex" => api::harness::Variant::Codex(api::harness::Codex {}),
_ => return None,
};
Some(api::Harness {
variant: Some(variant),
})
}
impl TryFrom<RunAgentsResult> for api::request::input::tool_call_result::Result {
type Error = ConvertToAPITypeError;
fn try_from(result: RunAgentsResult) -> Result<Self, Self::Error> {
match result {
RunAgentsResult::Launched {
model_id,
harness_type,
execution_mode,
agents,
} => Ok(
api::request::input::tool_call_result::Result::RunAgentsResult(
api::RunAgentsResult {
outcome: Some(api::run_agents_result::Outcome::Launched(
api::run_agents_result::Launched {
resolved_model_id: model_id,
resolved_harness: build_api_harness(&harness_type),
resolved_execution_mode: Some(execution_mode.into()),
agents: agents.into_iter().map(Into::into).collect(),
},
)),
},
),
),
RunAgentsResult::Denied { reason } => Ok(
api::request::input::tool_call_result::Result::RunAgentsResult(
api::RunAgentsResult {
outcome: Some(api::run_agents_result::Outcome::Denied(
api::run_agents_result::Denied { reason },
)),
},
),
),
RunAgentsResult::Failure { error } => Ok(
api::request::input::tool_call_result::Result::RunAgentsResult(
api::RunAgentsResult {
outcome: Some(api::run_agents_result::Outcome::Failure(
api::run_agents_result::Failure { error },
)),
},
),
),
// Reject is conveyed by the generic ToolCallResult.Cancel marker
// synthesized server-side on the next user input; nothing for the
// client to send on the wire here.
RunAgentsResult::Cancelled => Err(ConvertToAPITypeError::Ignore),
}
}
}
impl TryFrom<InsertReviewCommentsResult> for api::request::input::tool_call_result::Result {
type Error = ConvertToAPITypeError;
@@ -1504,6 +1670,22 @@ impl TryFrom<InsertReviewCommentsResult> for api::request::input::tool_call_resu
}
}
impl TryFrom<WaitForEventsResult> for api::request::input::tool_call_result::Result {
type Error = ConvertToAPITypeError;
/// Completed → wire form; Cancelled → drop (mirrors RunAgents).
fn try_from(result: WaitForEventsResult) -> Result<Self, Self::Error> {
match result {
WaitForEventsResult::Completed => Ok(
api::request::input::tool_call_result::Result::WaitForEvents(
api::WaitForEventsResult {},
),
),
WaitForEventsResult::Cancelled => Err(ConvertToAPITypeError::Ignore),
}
}
}
#[cfg(test)]
#[path = "convert_tests.rs"]
mod tests;
+142 -9
View File
@@ -1,17 +1,16 @@
mod convert;
use std::{fmt::Display, ops::Range, time::SystemTime};
use std::fmt::Display;
use std::ops::Range;
use std::time::SystemTime;
use galaxy_core::command::ExitCode;
use galaxy_terminal::model::BlockId;
use chrono::{DateTime, Local};
use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use warp_multi_agent_api::apply_file_diffs_result::success::UpdatedFileContent;
use crate::{
agent::FileLocations,
document::{AIDocumentId, AIDocumentVersion},
};
use crate::agent::FileLocations;
use crate::document::{AIDocumentId, AIDocumentVersion};
#[derive(Debug, Clone, PartialEq)]
pub enum AIAgentActionResultType {
@@ -95,6 +94,14 @@ pub enum AIAgentActionResultType {
TransferShellCommandControlToUser(TransferShellCommandControlToUserResult),
/// The result of asking the user a question.
AskUserQuestion(AskUserQuestionResult),
/// The result of an orchestrate tool call: launched (with per-agent
/// outcomes), launch denied (Stage 2), failure, or cancelled.
RunAgents(RunAgentsResult),
/// Result of the client-side wait_for_events watchdog or inbound
/// resume.
WaitForEvents(WaitForEventsResult),
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
@@ -161,6 +168,8 @@ impl Display for AIAgentActionResultType {
AIAgentActionResultType::SendMessageToAgent(result) => result.fmt(f),
AIAgentActionResultType::TransferShellCommandControlToUser(result) => result.fmt(f),
AIAgentActionResultType::AskUserQuestion(result) => result.fmt(f),
AIAgentActionResultType::RunAgents(result) => result.fmt(f),
AIAgentActionResultType::WaitForEvents(result) => result.fmt(f),
AIAgentActionResultType::OpenCodeReview | AIAgentActionResultType::InitProject => {
Ok(())
}
@@ -175,6 +184,8 @@ pub enum RequestCommandOutputResult {
command: String,
output: String,
exit_code: ExitCode,
start_ts: Option<DateTime<Local>>,
completed_ts: Option<DateTime<Local>>,
},
LongRunningCommandSnapshot {
block_id: BlockId,
@@ -266,6 +277,8 @@ pub enum WriteToLongRunningShellCommandResult {
block_id: BlockId,
output: String,
exit_code: ExitCode,
start_ts: Option<DateTime<Local>>,
completed_ts: Option<DateTime<Local>>,
},
Cancelled,
Error(ShellCommandError),
@@ -555,6 +568,8 @@ pub enum ReadShellCommandOutputResult {
block_id: BlockId,
output: String,
exit_code: ExitCode,
start_ts: Option<DateTime<Local>>,
completed_ts: Option<DateTime<Local>>,
},
LongRunningCommandSnapshot {
command: String,
@@ -763,6 +778,12 @@ impl AIAgentActionResultType {
AIAgentActionResultType::AskUserQuestion(_) => {
"The user's answers to clarifying questions"
}
AIAgentActionResultType::RunAgents(_) => {
"The result of an orchestrate batch of child agents"
}
AIAgentActionResultType::WaitForEvents(_) => {
"The local watchdog timed out while waiting for inbound events"
}
}
}
@@ -800,6 +821,8 @@ impl AIAgentActionResultType {
| TransferShellCommandControlToUserResult::CommandFinished { .. },
) => true,
Self::AskUserQuestion(AskUserQuestionResult::Success { .. }) => true,
Self::RunAgents(RunAgentsResult::Launched { .. }) => true,
Self::WaitForEvents(WaitForEventsResult::Completed) => true,
_ => false,
}
}
@@ -828,7 +851,10 @@ impl AIAgentActionResultType {
| Self::AskUserQuestion(AskUserQuestionResult::Error(_))
| Self::TransferShellCommandControlToUser(
TransferShellCommandControlToUserResult::Error(_),
) => true,
)
| Self::RunAgents(RunAgentsResult::Failure { .. } | RunAgentsResult::Denied { .. }) => {
true
}
_ => false,
}
}
@@ -952,7 +978,9 @@ impl AIAgentActionResultType {
| Self::StartAgent(StartAgentResult::Cancelled { .. })
| Self::SendMessageToAgent(SendMessageToAgentResult::Cancelled)
// SkippedByAutoApprove is intentionally excluded: the agent should continue.
| Self::AskUserQuestion(AskUserQuestionResult::Cancelled) => true,
| Self::AskUserQuestion(AskUserQuestionResult::Cancelled)
| Self::RunAgents(RunAgentsResult::Cancelled)
| Self::WaitForEvents(WaitForEventsResult::Cancelled) => true,
_ => false,
}
}
@@ -1234,6 +1262,8 @@ pub enum RequestComputerUseResult {
Approved {
screenshot: computer_use::Screenshot,
platform: computer_use::Platform,
/// The on-screen windows the agent may target.
windows: Vec<computer_use::WindowInfo>,
},
/// Request errored.
Error(String),
@@ -1280,6 +1310,9 @@ impl Display for FetchConversationResult {
}
}
// TODO(QUALITY-788): Delete legacy start_agent/start_agent_v2 result support once
// old preview orchestration history no longer needs parse/display/result compatibility.
// Linear issue: QUALITY-788.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum StartAgentResult {
Success {
@@ -1321,6 +1354,83 @@ impl Display for StartAgentResult {
}
}
/// The terminal outcome of an orchestrate tool call.
///
/// Mirrors the proto `RunAgentsResult` oneof, with an additional
/// `Cancelled` variant used internally by the action machinery when the
/// user clicks Reject. The proto wire form for cancellation is the
/// generic `ToolCallResult.Cancel` marker; the conversion code emits
/// `ConvertToAPITypeError::Ignore` for `Cancelled` so the input
/// interceptor can synthesize the marker on the next outbound input.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum RunAgentsResult {
/// Orchestration launched. Carries the resolved configuration and one
/// `AgentOutcome` per `agent_run_configs[]` entry, in input order.
Launched {
model_id: String,
harness_type: String,
execution_mode: RunAgentsLaunchedExecutionMode,
agents: Vec<RunAgentsAgentOutcome>,
},
/// Declined for a non-error reason (currently disapproval).
Denied { reason: String },
/// Actual error path: server-side validation rejected the call, or the
/// client could not begin the launch sequence at all.
Failure { error: String },
/// User rejected via the Reject button. Wire form is the generic
/// `ToolCallResult.Cancel` marker, synthesized by the server's input
/// interceptor on the next user input.
Cancelled,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum RunAgentsLaunchedExecutionMode {
Local,
Remote {
environment_id: String,
worker_host: String,
computer_use_enabled: bool,
},
}
/// Per-agent outcome reported in `RunAgentsResult::Launched.agents`.
/// Order mirrors the input order of `RunAgents.agent_run_configs[]`,
/// regardless of which `CreateAgentTask` call returned first.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RunAgentsAgentOutcome {
pub name: String,
pub kind: RunAgentsAgentOutcomeKind,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum RunAgentsAgentOutcomeKind {
Launched { agent_id: String },
Failed { error: String },
}
impl Display for RunAgentsResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RunAgentsResult::Launched { agents, .. } => {
let launched = agents
.iter()
.filter(|a| matches!(a.kind, RunAgentsAgentOutcomeKind::Launched { .. }))
.count();
write!(
f,
"Orchestrate launched ({launched}/{} agents started)",
agents.len()
)
}
RunAgentsResult::Denied { reason } => {
write!(f, "Orchestrate launch denied: {reason}")
}
RunAgentsResult::Failure { error } => write!(f, "Orchestrate failure: {error}"),
RunAgentsResult::Cancelled => write!(f, "Orchestrate cancelled"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum SendMessageToAgentResult {
Success { message_id: String },
@@ -1353,6 +1463,8 @@ pub enum TransferShellCommandControlToUserResult {
block_id: BlockId,
output: String,
exit_code: ExitCode,
start_ts: Option<DateTime<Local>>,
completed_ts: Option<DateTime<Local>>,
},
Cancelled,
Error(ShellCommandError),
@@ -1447,3 +1559,24 @@ impl Display for AskUserQuestionResult {
}
}
}
/// Result of a client-side wait_for_events action.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub enum WaitForEventsResult {
/// Watchdog fired or an inbound resume signal closed the wait. The
/// agent's next turn observes an empty WaitForEvents result on the
/// wire and decides how to proceed.
Completed,
/// User cancelled the conversation while waiting. Mirrors
/// RunAgents::Cancelled: no tool-call result is sent on the wire.
Cancelled,
}
impl Display for WaitForEventsResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Completed => write!(f, "Wait for events completed"),
Self::Cancelled => write!(f, "Wait for events cancelled"),
}
}
}
+23 -3
View File
@@ -5,9 +5,22 @@ use warp_multi_agent_api as api;
/// A citation listed in an AI response.
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub enum AIAgentCitation {
WarpDriveObject { uid: String },
WarpDocumentation { path: String },
WebPage { url: String },
WarpDriveObject {
uid: String,
},
WarpDocumentation {
path: String,
},
WebPage {
url: String,
},
/// A memory from an attached memory store. `content` is the raw memory
/// text shown as a preview in the chip.
AgentMemory {
memory_store_id: String,
memory_id: String,
content: String,
},
}
impl Display for AIAgentCitation {
@@ -22,6 +35,13 @@ impl Display for AIAgentCitation {
AIAgentCitation::WebPage { url } => {
write!(f, "Web Page: {url}")
}
AIAgentCitation::AgentMemory {
memory_store_id,
memory_id,
..
} => {
write!(f, "Agent Memory: {memory_store_id}/{memory_id}")
}
}
}
}
+2 -1
View File
@@ -5,7 +5,8 @@ use galaxy_terminal::shell::ShellLaunchData;
use itertools::Itertools as _;
use crate::agent::action_result::FileContext;
use crate::{index::locations::CodeContextLocation, paths::shell_native_absolute_path};
use crate::index::locations::CodeContextLocation;
use crate::paths::shell_native_absolute_path;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FileLocations {
+1
View File
@@ -3,6 +3,7 @@ pub mod action_result;
mod citation;
pub mod convert;
pub mod file_locations;
pub mod orchestration_config;
pub use citation::{AIAgentCitation, UnknownCitationTypeError};
pub use file_locations::{group_file_contexts_for_display, FileLocations};
+218
View File
@@ -0,0 +1,218 @@
use warp_multi_agent_api as api;
use super::action::RunAgentsRequest;
/// Client-side representation of the orchestration config attached to a
/// conversation via `OrchestrationConfigSnapshot`.
///
/// Mirrors the proto `OrchestrationConfig` but uses Rust-native types
/// to keep view / model code free of proto imports.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct OrchestrationConfig {
pub model_id: String,
pub harness_type: String,
pub execution_mode: OrchestrationExecutionMode,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum OrchestrationExecutionMode {
Local,
Remote {
environment_id: String,
worker_host: String,
},
}
impl OrchestrationExecutionMode {
pub fn is_remote(&self) -> bool {
matches!(self, Self::Remote { .. })
}
}
/// User's approval state for orchestration on the active config.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)]
pub enum OrchestrationConfigStatus {
/// No `OrchestrationConfigSnapshot` has been seen yet.
#[default]
None,
Approved,
Disapproved,
}
impl OrchestrationConfigStatus {
pub fn is_approved(&self) -> bool {
matches!(self, Self::Approved)
}
pub fn is_disapproved(&self) -> bool {
matches!(self, Self::Disapproved)
}
}
// ---------------------------------------------------------------------------
// Match check — determines whether a `run_agents` call auto-launches.
// ---------------------------------------------------------------------------
/// Returns `true` when the `run_agents` call's run-wide fields match
/// the active approved `OrchestrationConfig`, meaning the confirmation
/// card can be skipped (auto-launch).
///
/// Empty/unset fields on the call are treated as inheriting from the
/// config (and therefore matching). Fields not in the config
/// (`computer_use_enabled`, `skills`, `base_prompt`, `agent_run_configs`,
/// per-agent `title`) are excluded from the check.
pub fn matches_active_config(request: &RunAgentsRequest, config: &OrchestrationConfig) -> bool {
// model_id — empty on the call means "inherit from config" → matches.
if !request.model_id.is_empty() && request.model_id != config.model_id {
return false;
}
// harness_type
if !request.harness_type.is_empty() && request.harness_type != config.harness_type {
return false;
}
// execution_mode variant must agree.
match (&request.execution_mode, &config.execution_mode) {
(super::action::RunAgentsExecutionMode::Local, OrchestrationExecutionMode::Local) => true,
(
super::action::RunAgentsExecutionMode::Remote {
environment_id,
worker_host,
..
},
OrchestrationExecutionMode::Remote {
environment_id: cfg_env,
worker_host: cfg_host,
},
) => {
let env_matches = environment_id.is_empty() || environment_id == cfg_env;
let host_matches = worker_host.is_empty() || worker_host == cfg_host;
env_matches && host_matches
}
// Variant mismatch (Local vs Remote).
_ => false,
}
}
// ---------------------------------------------------------------------------
// Proto ↔ native conversions
// ---------------------------------------------------------------------------
impl OrchestrationConfig {
/// Converts from the proto `OrchestrationConfig` message.
pub fn from_proto(proto: &api::OrchestrationConfig) -> Self {
let execution_mode = match &proto.execution_mode {
Some(api::orchestration_config::ExecutionMode::Remote(remote)) => {
OrchestrationExecutionMode::Remote {
environment_id: remote.environment_id.clone(),
worker_host: remote.worker_host.clone(),
}
}
Some(api::orchestration_config::ExecutionMode::Local(_)) | None => {
OrchestrationExecutionMode::Local
}
};
let harness_type = harness_proto_to_string(proto.harness.as_ref()).unwrap_or_default();
Self {
model_id: proto.model_id.clone(),
harness_type,
execution_mode,
}
}
/// Converts to the proto `OrchestrationConfig` message.
pub fn to_proto(&self) -> api::OrchestrationConfig {
let execution_mode = match &self.execution_mode {
OrchestrationExecutionMode::Local => {
Some(api::orchestration_config::ExecutionMode::Local(
api::orchestration_config::Local {},
))
}
OrchestrationExecutionMode::Remote {
environment_id,
worker_host,
} => Some(api::orchestration_config::ExecutionMode::Remote(
api::orchestration_config::Remote {
environment_id: environment_id.clone(),
worker_host: worker_host.clone(),
},
)),
};
api::OrchestrationConfig {
model_id: self.model_id.clone(),
harness: harness_type_to_proto(&self.harness_type),
execution_mode,
}
}
}
impl OrchestrationConfigStatus {
/// Converts from the proto `OrchestrationStatus` message.
pub fn from_proto(proto: Option<&api::OrchestrationStatus>) -> Self {
let Some(status) = proto else {
return Self::None;
};
match &status.status {
Some(api::orchestration_status::Status::Approved(_)) => Self::Approved,
Some(api::orchestration_status::Status::Disapproved(_)) => Self::Disapproved,
None => Self::None,
}
}
/// Converts to the proto `OrchestrationStatus` message.
pub fn to_proto(&self) -> Option<api::OrchestrationStatus> {
match self {
Self::None => None,
Self::Approved => Some(api::OrchestrationStatus {
status: Some(api::orchestration_status::Status::Approved(
api::orchestration_status::Approved {},
)),
}),
Self::Disapproved => Some(api::OrchestrationStatus {
status: Some(api::orchestration_status::Status::Disapproved(
api::orchestration_status::Disapproved {},
)),
}),
}
}
}
/// Maps the proto `Harness` oneof to a client-side string identifier.
/// Returns `None` for an unset variant.
fn harness_proto_to_string(harness: Option<&api::Harness>) -> Option<String> {
let variant = harness?.variant.as_ref()?;
Some(
match variant {
api::harness::Variant::Oz(_) => "oz",
api::harness::Variant::ClaudeCode(_) => "claude",
api::harness::Variant::OpenCode(_) => "opencode",
api::harness::Variant::Gemini(_) => "gemini",
api::harness::Variant::Codex(_) => "codex",
}
.to_string(),
)
}
/// Converts a client-side harness string identifier to the proto `Harness`
/// oneof variant. Returns `None` for empty or unknown strings.
fn harness_type_to_proto(harness_type: &str) -> Option<api::Harness> {
let variant = match harness_type {
"oz" => api::harness::Variant::Oz(api::harness::Oz {}),
"claude" => api::harness::Variant::ClaudeCode(api::harness::ClaudeCode {}),
"opencode" => api::harness::Variant::OpenCode(api::harness::OpenCode {}),
"gemini" => api::harness::Variant::Gemini(api::harness::Gemini {}),
"codex" => api::harness::Variant::Codex(api::harness::Codex {}),
_ => return None,
};
Some(api::Harness {
variant: Some(variant),
})
}
#[cfg(test)]
#[path = "orchestration_config_tests.rs"]
mod tests;
@@ -0,0 +1,180 @@
use super::*;
use crate::agent::action::{RunAgentsAgentRunConfig, RunAgentsExecutionMode, RunAgentsRequest};
fn make_config(model: &str, harness: &str, remote: bool) -> OrchestrationConfig {
OrchestrationConfig {
model_id: model.to_string(),
harness_type: harness.to_string(),
execution_mode: if remote {
OrchestrationExecutionMode::Remote {
environment_id: "env-1".to_string(),
worker_host: "warp".to_string(),
}
} else {
OrchestrationExecutionMode::Local
},
}
}
fn make_request(model: &str, harness: &str, remote: bool) -> RunAgentsRequest {
RunAgentsRequest {
summary: "test".to_string(),
base_prompt: "prompt".to_string(),
skills: vec![],
model_id: model.to_string(),
harness_type: harness.to_string(),
execution_mode: if remote {
RunAgentsExecutionMode::Remote {
environment_id: "env-1".to_string(),
worker_host: "warp".to_string(),
computer_use_enabled: false,
}
} else {
RunAgentsExecutionMode::Local
},
agent_run_configs: vec![RunAgentsAgentRunConfig {
name: "a".to_string(),
prompt: String::new(),
title: String::new(),
}],
plan_id: String::new(),
harness_auth_secret_name: None,
}
}
#[test]
fn exact_match_local() {
let config = make_config("auto", "oz", false);
let request = make_request("auto", "oz", false);
assert!(matches_active_config(&request, &config));
}
#[test]
fn exact_match_remote() {
let config = make_config("auto", "oz", true);
let request = make_request("auto", "oz", true);
assert!(matches_active_config(&request, &config));
}
#[test]
fn empty_model_inherits_and_matches() {
let config = make_config("auto", "oz", false);
let request = make_request("", "oz", false);
assert!(matches_active_config(&request, &config));
}
#[test]
fn empty_harness_inherits_and_matches() {
let config = make_config("auto", "oz", false);
let request = make_request("auto", "", false);
assert!(matches_active_config(&request, &config));
}
#[test]
fn different_model_mismatches() {
let config = make_config("auto", "oz", false);
let request = make_request("claude-4-6-opus-high", "oz", false);
assert!(!matches_active_config(&request, &config));
}
#[test]
fn different_harness_mismatches() {
let config = make_config("auto", "oz", false);
let request = make_request("auto", "claude", false);
assert!(!matches_active_config(&request, &config));
}
#[test]
fn execution_mode_variant_mismatch() {
let config = make_config("auto", "oz", false);
let request = make_request("auto", "oz", true);
assert!(!matches_active_config(&request, &config));
}
#[test]
fn remote_different_environment_mismatches() {
let config = make_config("auto", "oz", true);
let mut request = make_request("auto", "oz", true);
if let RunAgentsExecutionMode::Remote {
ref mut environment_id,
..
} = request.execution_mode
{
*environment_id = "env-other".to_string();
}
assert!(!matches_active_config(&request, &config));
}
#[test]
fn remote_empty_env_inherits_and_matches() {
let config = make_config("auto", "oz", true);
let mut request = make_request("auto", "oz", true);
if let RunAgentsExecutionMode::Remote {
ref mut environment_id,
..
} = request.execution_mode
{
*environment_id = String::new();
}
assert!(matches_active_config(&request, &config));
}
#[test]
fn computer_use_not_in_match_check() {
let config = make_config("auto", "oz", true);
let mut request = make_request("auto", "oz", true);
if let RunAgentsExecutionMode::Remote {
ref mut computer_use_enabled,
..
} = request.execution_mode
{
*computer_use_enabled = true;
}
// computer_use_enabled differs but should still match
assert!(matches_active_config(&request, &config));
}
#[test]
fn status_default_is_none() {
assert_eq!(
OrchestrationConfigStatus::default(),
OrchestrationConfigStatus::None
);
}
#[test]
fn status_predicates() {
assert!(OrchestrationConfigStatus::Approved.is_approved());
assert!(!OrchestrationConfigStatus::Approved.is_disapproved());
assert!(OrchestrationConfigStatus::Disapproved.is_disapproved());
assert!(!OrchestrationConfigStatus::None.is_approved());
}
#[test]
fn proto_round_trip_config_local() {
let config = make_config("auto", "oz", false);
let proto = config.to_proto();
let round_tripped = OrchestrationConfig::from_proto(&proto);
assert_eq!(config, round_tripped);
}
#[test]
fn proto_round_trip_config_remote() {
let config = make_config("auto", "claude", true);
let proto = config.to_proto();
let round_tripped = OrchestrationConfig::from_proto(&proto);
assert_eq!(config, round_tripped);
}
#[test]
fn proto_round_trip_status() {
for status in [
OrchestrationConfigStatus::None,
OrchestrationConfigStatus::Approved,
OrchestrationConfigStatus::Disapproved,
] {
let proto = status.to_proto();
let round_tripped = OrchestrationConfigStatus::from_proto(proto.as_ref());
assert_eq!(status, round_tripped);
}
}
+437 -13
View File
@@ -1,11 +1,24 @@
pub use crate::aws_credentials::{AwsCredentials, AwsCredentialsState};
use galaxyui::{Entity, ModelContext, SingletonEntity};
use galaxyui_extras::secure_storage::{self, AppContextExt};
use std::time::{Duration, SystemTime};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use warp_multi_agent_api as api;
use galaxyui_core::{Entity, ModelContext, SingletonEntity};
use galaxyui_extras::secure_storage::{self, AppContextExt};
pub use crate::aws_credentials::{AwsCredentials, AwsCredentialsState};
pub use crate::geap_credentials::{
GeapCredentials, GeapCredentialsState, GeapFederation, GeapMintBinding,
LoadGeapCredentialsError, GEAP_REFRESH_LEAD_TIME,
};
const SECURE_STORAGE_KEY: &str = "AiApiKeys";
/// Secure-storage key for the connected xAI/Grok subscription's OAuth tokens.
/// Kept separate from [`SECURE_STORAGE_KEY`] because these are OAuth tokens with
/// a refresh lifecycle, not a user-pasted static key.
const GROK_SECURE_STORAGE_KEY: &str = "GrokOAuthTokens";
/// Emitted when user-provided API keys are updated in-memory.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ApiKeyManagerEvent {
@@ -17,11 +30,44 @@ pub enum ApiKeyManagerEvent {
/// These are used for "Bring Your Own API Key" functionality, allowing
/// users to use their own API keys instead of Warp's.
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ApiKeys {
pub google: Option<String>,
pub anthropic: Option<String>,
pub openai: Option<String>,
pub open_router: Option<String>,
pub custom_endpoints: Vec<CustomEndpoint>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct CustomEndpoint {
pub name: String,
pub url: String,
pub api_key: String,
pub models: Vec<CustomEndpointModel>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct CustomEndpointModel {
pub name: String,
pub alias: Option<String>,
/// Stable identifier used as `ModelConfig.{base,coding,cli_agent,computer_use_agent}` and
/// as the `CustomModelProviders.providers[*].models[*].config_key` on the request wire.
/// Generated as a UUIDv4 at model creation.
pub config_key: String,
}
impl CustomEndpointModel {
/// Picker label: prefer the user-provided alias; fall back to the raw model name
/// so a row is never blank.
pub fn display_label(&self) -> &str {
match self.alias.as_deref() {
Some(alias) if !alias.trim().is_empty() => alias,
_ => &self.name,
}
}
}
impl ApiKeys {
@@ -30,6 +76,70 @@ impl ApiKeys {
|| self.anthropic.is_some()
|| self.google.is_some()
|| self.open_router.is_some()
|| self
.custom_endpoints
.iter()
.any(|endpoint| !endpoint.api_key.trim().is_empty())
}
/// Number of single-provider API keys currently configured (OpenAI,
/// Anthropic, Google, OpenRouter). Custom endpoints are counted separately
/// via `custom_endpoints`.
pub fn provider_key_count(&self) -> usize {
[
&self.openai,
&self.anthropic,
&self.google,
&self.open_router,
]
.into_iter()
.filter(|key| key.as_deref().is_some_and(|v| !v.trim().is_empty()))
.count()
}
}
/// OAuth tokens for a connected xAI / Grok subscription (e.g. SuperGrok).
///
/// Persisted to secure storage under [`GROK_SECURE_STORAGE_KEY`], separate from
/// the BYO [`ApiKeys`] blob because these are OAuth tokens with a refresh
/// lifecycle rather than a user-pasted static key. `crate::grok_subscription`
/// owns refreshing them; this module is the storage and request-injection
/// source of truth that [`ApiKeyManager::api_keys_for_request`] reads from.
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct GrokTokens {
pub access_token: String,
#[serde(default)]
pub refresh_token: Option<String>,
/// Absolute time at which `access_token` expires, if the provider told us.
#[serde(default)]
pub expires_at: Option<SystemTime>,
/// When the user originally connected the subscription (i.e. when the
/// browser OAuth flow completed). Carried over across token refreshes so
/// it keeps reflecting the initial connection, not the latest refresh;
/// surfaced in the settings UI as "Connected on ...". `None` for tokens
/// stored before this field existed.
#[serde(default)]
pub connected_at: Option<SystemTime>,
}
impl GrokTokens {
/// Returns the access token whenever it is non-empty, regardless of
/// expiry. Possibly-expired tokens are still sent so the server stays the
/// final authority on token validity (it rejects truly invalid tokens);
/// `crate::grok_subscription` refreshes (nearly) expired tokens in the
/// background.
pub fn access_token_for_request(&self) -> Option<&str> {
(!self.access_token.trim().is_empty()).then_some(self.access_token.as_str())
}
/// Returns `true` when the token is known to expire within `lead_time` and
/// should be proactively refreshed. Tokens with an unknown expiry never
/// report as needing a refresh (there's no expiry signal to act on).
pub fn needs_refresh(&self, lead_time: Duration) -> bool {
match self.expires_at {
Some(expires_at) => expires_at <= SystemTime::now() + lead_time,
None => false,
}
}
}
@@ -41,27 +151,55 @@ pub enum AwsCredentialsRefreshStrategy {
LocalChain,
/// Credentials are managed externally via OIDC/STS.
/// The task ID is used to scope the STS AssumeRoleWithWebIdentity session.
/// The role ARN is the IAM role to assume via STS.
/// The role ARN + region are the info used to assume the IAM role via STS.
OidcManaged {
task_id: Option<String>,
role_arn: String,
region: String,
},
}
/// A structure that manages API keys for AI providers.
pub struct ApiKeyManager {
keys: ApiKeys,
/// OAuth tokens for a connected xAI/Grok subscription, if any. Persisted
/// separately from `keys` under [`GROK_SECURE_STORAGE_KEY`];
/// `crate::grok_subscription` keeps these fresh.
grok_tokens: Option<GrokTokens>,
/// Whether background refresh of `grok_tokens` is currently allowed.
/// Mirrors the BYO API key policy, which lives in the app layer; wired in
/// via `ApiKeyManager::set_grok_refresh_allowed` (`crate::grok_subscription`).
#[cfg(not(target_family = "wasm"))]
pub(crate) grok_refresh_allowed: bool,
/// Guards against overlapping Grok token refreshes: the proactive refresh
/// timer and the request-time safety net
/// (`ApiKeyManager::refresh_grok_tokens_if_needed`) can otherwise race.
#[cfg(not(target_family = "wasm"))]
pub(crate) grok_refresh_in_flight: bool,
pub(crate) aws_credentials_state: AwsCredentialsState,
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy,
/// In-memory Gemini Enterprise (GEAP) credential state.
pub(crate) geap_credentials_state: GeapCredentialsState,
secure_storage_write_version: u64,
grok_secure_storage_write_version: u64,
}
impl ApiKeyManager {
pub fn new(ctx: &mut ModelContext<Self>) -> Self {
let keys = Self::load_keys_from_secure_storage(ctx);
let grok_tokens = Self::load_grok_tokens_from_secure_storage(ctx);
Self {
keys,
grok_tokens,
#[cfg(not(target_family = "wasm"))]
grok_refresh_allowed: false,
#[cfg(not(target_family = "wasm"))]
grok_refresh_in_flight: false,
aws_credentials_state: AwsCredentialsState::Missing,
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy::default(),
geap_credentials_state: GeapCredentialsState::Missing,
secure_storage_write_version: 0,
grok_secure_storage_write_version: 0,
}
}
@@ -69,6 +207,39 @@ impl ApiKeyManager {
&self.keys
}
/// The currently stored xAI/Grok OAuth tokens, if the user has connected a
/// Grok subscription.
pub fn grok_tokens(&self) -> Option<&GrokTokens> {
self.grok_tokens.as_ref()
}
/// Returns `true` when a Grok subscription is connected with a usable OAuth
/// access token.
pub fn has_grok_subscription(&self) -> bool {
self.grok_tokens
.as_ref()
.and_then(GrokTokens::access_token_for_request)
.is_some()
}
/// Returns `true` when the user has any usable BYO credential: a pasted
/// provider or custom-endpoint key, or a connected Grok subscription.
pub fn has_any_key(&self) -> bool {
self.keys.has_any_key() || self.has_grok_subscription()
}
/// Stores (or clears, with `None`) the xAI/Grok OAuth tokens and persists
/// them to secure storage. No-op when the value is unchanged so we don't
/// emit spurious events or schedule redundant keychain writes.
pub fn set_grok_tokens(&mut self, tokens: Option<GrokTokens>, ctx: &mut ModelContext<Self>) {
if self.grok_tokens == tokens {
return;
}
self.grok_tokens = tokens;
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
self.write_grok_tokens_to_secure_storage(ctx);
}
pub fn set_google_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
self.keys.google = key;
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
@@ -93,6 +264,82 @@ impl ApiKeyManager {
self.write_keys_to_secure_storage(ctx);
}
pub fn add_custom_endpoint(
&mut self,
name: String,
url: String,
api_key: String,
models: Vec<(String, Option<String>, Option<String>)>,
ctx: &mut ModelContext<Self>,
) {
self.keys.custom_endpoints.push(CustomEndpoint {
name,
url,
api_key,
models: models
.into_iter()
.map(|(name, alias, config_key)| CustomEndpointModel {
name,
alias,
config_key: config_key
.filter(|k| !k.is_empty())
.unwrap_or_else(|| Uuid::new_v4().to_string()),
})
.collect(),
});
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
self.write_keys_to_secure_storage(ctx);
}
pub fn save_custom_endpoint(
&mut self,
index: usize,
name: String,
url: String,
api_key: String,
models: Vec<(String, Option<String>, Option<String>)>,
ctx: &mut ModelContext<Self>,
) {
if index >= self.keys.custom_endpoints.len() {
return;
}
self.keys.custom_endpoints[index] = CustomEndpoint {
name,
url,
api_key,
models: models
.into_iter()
.map(|(name, alias, config_key)| CustomEndpointModel {
name,
alias,
config_key: config_key
.filter(|k| !k.is_empty())
.unwrap_or_else(|| Uuid::new_v4().to_string()),
})
.collect(),
};
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
self.write_keys_to_secure_storage(ctx);
}
pub fn remove_custom_endpoint(&mut self, index: usize, ctx: &mut ModelContext<Self>) {
if index >= self.keys.custom_endpoints.len() {
return;
}
self.keys.custom_endpoints.remove(index);
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
self.write_keys_to_secure_storage(ctx);
}
pub fn clear_custom_endpoints(&mut self, ctx: &mut ModelContext<Self>) {
if self.keys.custom_endpoints.is_empty() {
return;
}
self.keys.custom_endpoints.clear();
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
self.write_keys_to_secure_storage(ctx);
}
pub fn set_aws_credentials_state(
&mut self,
state: AwsCredentialsState,
@@ -106,6 +353,22 @@ impl ApiKeyManager {
&self.aws_credentials_state
}
pub fn set_geap_credentials_state(
&mut self,
state: GeapCredentialsState,
ctx: &mut ModelContext<Self>,
) {
if self.geap_credentials_state == state {
return;
}
self.geap_credentials_state = state;
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
}
pub fn geap_credentials_state(&self) -> &GeapCredentialsState {
&self.geap_credentials_state
}
pub fn aws_credentials_refresh_strategy(&self) -> AwsCredentialsRefreshStrategy {
self.aws_credentials_refresh_strategy.clone()
}
@@ -117,10 +380,60 @@ impl ApiKeyManager {
self.aws_credentials_refresh_strategy = strategy;
}
/// Builds the `CustomModelProviders` registry that ships with every agent request.
///
/// Emits one [`CustomModelProvider`] per configured [`CustomEndpoint`], each populated with
/// all of its [`CustomEndpointModel`]s. The per-model `config_key` is what the server uses
/// to map a `ModelConfig.{base,coding,cli_agent,computer_use_agent}` selection back to a
/// user-provided endpoint, so it MUST be the same UUID we store locally.
///
/// Returns `None` when custom models should not be included or no endpoint has both a
/// non-empty URL and API key.
pub fn custom_model_providers_for_request(
&self,
include_custom_models: bool,
) -> Option<api::request::settings::CustomModelProviders> {
if !include_custom_models {
return None;
}
let providers: Vec<_> = self
.keys
.custom_endpoints
.iter()
.filter(|endpoint| !endpoint.url.trim().is_empty() && !endpoint.api_key.is_empty())
.map(
|endpoint| api::request::settings::custom_model_providers::CustomModelProvider {
base_url: endpoint.url.clone(),
api_key: endpoint.api_key.clone(),
models: endpoint
.models
.iter()
.filter(|m| !m.name.trim().is_empty() && !m.config_key.is_empty())
.map(
|m| api::request::settings::custom_model_providers::CustomModel {
slug: m.name.clone(),
config_key: m.config_key.clone(),
},
)
.collect(),
},
)
.filter(|provider| !provider.models.is_empty())
.collect();
if providers.is_empty() {
None
} else {
Some(api::request::settings::CustomModelProviders { providers })
}
}
pub fn api_keys_for_request(
&self,
include_byo_keys: bool,
include_aws_bedrock_credentials: bool,
geap_binding: Option<GeapMintBinding>,
) -> Option<api::request::settings::ApiKeys> {
let anthropic = include_byo_keys
.then(|| self.keys.anthropic.clone())
@@ -138,6 +451,22 @@ impl ApiKeyManager {
.then(|| self.keys.open_router.clone())
.flatten()
.unwrap_or_default();
// The connected Grok subscription's OAuth access token is user-provided
// auth, just like a pasted BYO API key, so it respects the same BYO
// policy gate: when BYO keys are disabled (e.g. by workspace policy),
// the token must not be sent. Possibly-expired tokens ARE sent — the
// server is the authority on validity.
let grok_oauth_access_token = include_byo_keys
.then(|| {
self.grok_tokens
.as_ref()
.and_then(GrokTokens::access_token_for_request)
.map(str::to_owned)
})
.flatten()
.unwrap_or_default();
// Also include credentials when running with OIDC-managed Bedrock inference, regardless
// of the per-user setting flag (which only applies to the local credential chain path).
let include_aws = include_aws_bedrock_credentials
@@ -154,11 +483,36 @@ impl ApiKeyManager {
})
.flatten();
// Gemini Enterprise (GEAP) credentials attach only when the caller's
// gate is on AND the stored token was minted for that same
// (user, audience, SA) binding.
let google_cloud_credentials: Option<
api::request::settings::api_keys::GoogleCloudCredentials,
> = geap_binding
.as_ref()
.and_then(|binding| match self.geap_credentials_state {
GeapCredentialsState::Loaded {
ref credentials,
ref minted_for,
..
} if minted_for == binding => credentials
.access_token_for_request()
.map(|_| credentials.clone().into()),
GeapCredentialsState::Refreshing {
previous: Some((ref credentials, ref minted_for)),
} if minted_for == binding => credentials
.access_token_for_request()
.map(|_| credentials.clone().into()),
_ => None,
});
if anthropic.is_empty()
&& openai.is_empty()
&& google.is_empty()
&& open_router.is_empty()
&& grok_oauth_access_token.is_empty()
&& aws_credentials.is_none()
&& google_cloud_credentials.is_none()
{
None
} else {
@@ -167,8 +521,10 @@ impl ApiKeyManager {
openai,
google,
open_router,
grok_oauth_access_token,
allow_use_of_warp_credits: false,
aws_credentials,
google_cloud_credentials,
})
}
}
@@ -184,32 +540,96 @@ impl ApiKeyManager {
}
};
let keys = match serde_json::from_str(&key_json) {
match serde_json::from_str(&key_json) {
Ok(keys) => keys,
Err(e) => {
log::error!("Failed to deserialize API keys: {e:#}");
ApiKeys::default()
}
};
keys
}
}
fn write_keys_to_secure_storage(&mut self, ctx: &mut ModelContext<Self>) {
let keys = self.keys.clone();
let json = match serde_json::to_string(&keys) {
let json = match serde_json::to_string(&self.keys) {
Ok(json) => json,
Err(e) => {
log::error!("Failed to serialize API keys: {e:#}");
return;
}
};
self.secure_storage_write_version += 1;
let write_version = self.secure_storage_write_version;
if let Err(e) = ctx.secure_storage().write_value(SECURE_STORAGE_KEY, &json) {
log::error!("Failed to write API keys to secure storage: {e:#}");
// Defer the keychain write so it doesn't block the current event
// processing. The in-memory state is already updated and events
// already emitted, so the UI updates immediately while the
// potentially slow platform secure-storage call runs in a
// subsequent main-thread callback. Skip stale callbacks so older
// writes cannot complete after and overwrite a newer payload.
ctx.spawn(async move { json }, move |me, json, ctx| {
if write_version != me.secure_storage_write_version {
return;
}
if let Err(e) = ctx.secure_storage().write_value(SECURE_STORAGE_KEY, &json) {
log::error!("Failed to write API keys to secure storage: {e:#}");
}
});
}
fn load_grok_tokens_from_secure_storage(ctx: &mut ModelContext<Self>) -> Option<GrokTokens> {
let json = match ctx.secure_storage().read_value(GROK_SECURE_STORAGE_KEY) {
Ok(json) => json,
Err(e) => {
if !matches!(e, secure_storage::Error::NotFound) {
log::error!("Failed to read Grok tokens from secure storage: {e:#}");
}
return None;
}
};
match serde_json::from_str(&json) {
Ok(tokens) => Some(tokens),
Err(e) => {
log::error!("Failed to deserialize Grok tokens: {e:#}");
None
}
}
}
fn write_grok_tokens_to_secure_storage(&mut self, ctx: &mut ModelContext<Self>) {
// `Some(json)` writes the tokens; `None` removes the stored entry (the
// user disconnected). Serialize up front so the deferred callback only
// touches the keychain.
let payload = match self.grok_tokens.as_ref().map(serde_json::to_string) {
Some(Ok(json)) => Some(json),
Some(Err(e)) => {
log::error!("Failed to serialize Grok tokens: {e:#}");
return;
}
None => None,
};
self.grok_secure_storage_write_version += 1;
let write_version = self.grok_secure_storage_write_version;
// Defer the keychain write/remove like `write_keys_to_secure_storage`,
// skipping stale callbacks so an older write can't clobber a newer one.
ctx.spawn(async move { payload }, move |me, payload, ctx| {
if write_version != me.grok_secure_storage_write_version {
return;
}
let result = match payload {
Some(ref json) => ctx
.secure_storage()
.write_value(GROK_SECURE_STORAGE_KEY, json),
None => ctx.secure_storage().remove_value(GROK_SECURE_STORAGE_KEY),
};
if let Err(e) = result {
if !matches!(e, secure_storage::Error::NotFound) {
log::error!("Failed to persist Grok tokens to secure storage: {e:#}");
}
}
});
}
}
impl Entity for ApiKeyManager {
@@ -217,3 +637,7 @@ impl Entity for ApiKeyManager {
}
impl SingletonEntity for ApiKeyManager {}
#[cfg(test)]
#[path = "api_keys_tests.rs"]
mod tests;
+696
View File
@@ -0,0 +1,696 @@
use std::time::{Duration, SystemTime};
use super::*;
fn make_manager(keys: ApiKeys) -> ApiKeyManager {
make_manager_with_grok(keys, None)
}
fn make_manager_with_grok(keys: ApiKeys, grok_tokens: Option<GrokTokens>) -> ApiKeyManager {
ApiKeyManager {
keys,
grok_tokens,
#[cfg(not(target_family = "wasm"))]
grok_refresh_allowed: false,
#[cfg(not(target_family = "wasm"))]
grok_refresh_in_flight: false,
aws_credentials_state: AwsCredentialsState::Missing,
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy::default(),
geap_credentials_state: GeapCredentialsState::Missing,
secure_storage_write_version: 0,
grok_secure_storage_write_version: 0,
}
}
fn make_manager_with_geap(geap_credentials_state: GeapCredentialsState) -> ApiKeyManager {
let mut manager = make_manager(ApiKeys::default());
manager.geap_credentials_state = geap_credentials_state;
manager
}
fn grok_tokens(access_token: &str, expires_in: Option<u64>) -> GrokTokens {
GrokTokens {
access_token: access_token.into(),
refresh_token: Some("refresh".into()),
expires_at: expires_in.map(|secs| SystemTime::now() + Duration::from_secs(secs)),
connected_at: None,
}
}
fn geap_credentials(access_token: &str, expires_in: Option<u64>) -> GeapCredentials {
GeapCredentials::new(
access_token.into(),
expires_in.map(|secs| SystemTime::now() + Duration::from_secs(secs)),
)
}
fn geap_binding() -> GeapMintBinding {
GeapMintBinding {
user_uid: "user-1".into(),
audience:
"//iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/p/providers/q"
.into(),
federation: GeapFederation::ServiceAccount {
email: "sa@proj.iam.gserviceaccount.com".into(),
},
}
}
// The expected binding the request build site passes in is the same type as
// the stored `minted_for`, so the attach check is a plain `==`.
fn geap_gate() -> GeapMintBinding {
geap_binding()
}
fn geap_loaded(access_token: &str, expires_in: Option<u64>) -> GeapCredentialsState {
GeapCredentialsState::Loaded {
credentials: geap_credentials(access_token, expires_in),
loaded_at: SystemTime::now(),
minted_for: geap_binding(),
}
}
fn endpoint(
name: &str,
url: &str,
api_key: &str,
models: &[(&str, Option<&str>)],
) -> CustomEndpoint {
endpoint_with_keys(
name,
url,
api_key,
&models
.iter()
.enumerate()
.map(|(i, (n, a))| (*n, *a, format!("cfg-{i}")))
.collect::<Vec<_>>()
.iter()
.map(|(n, a, k)| (*n, *a, k.as_str()))
.collect::<Vec<_>>(),
)
}
fn endpoint_with_keys(
name: &str,
url: &str,
api_key: &str,
models: &[(&str, Option<&str>, &str)],
) -> CustomEndpoint {
CustomEndpoint {
name: name.into(),
url: url.into(),
api_key: api_key.into(),
models: models
.iter()
.map(|(n, a, cfg)| CustomEndpointModel {
name: (*n).into(),
alias: a.map(|s| s.into()),
config_key: (*cfg).into(),
})
.collect(),
}
}
// ── serde round-trip ────────────────────────────────────────────
#[test]
fn serde_round_trip_empty() {
let keys = ApiKeys::default();
let json = serde_json::to_string(&keys).unwrap();
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
assert_eq!(keys, deser);
}
#[test]
fn serde_round_trip_with_provider_keys() {
let keys = ApiKeys {
openai: Some("sk-openai".into()),
anthropic: Some("sk-ant-abc".into()),
google: Some("AIzaSy123".into()),
open_router: Some("sk-or-xxx".into()),
custom_endpoints: vec![],
};
let json = serde_json::to_string(&keys).unwrap();
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
assert_eq!(keys, deser);
}
#[test]
fn serde_round_trip_with_custom_endpoints() {
let keys = ApiKeys {
openai: None,
anthropic: None,
google: None,
open_router: None,
custom_endpoints: vec![
endpoint("ep1", "https://a.io/v1", "key1", &[("gpt-4", Some("fast"))]),
endpoint(
"ep2",
"https://b.io/v1",
"key2",
&[("llama-70b", None), ("mixtral", Some("mix"))],
),
],
};
let json = serde_json::to_string(&keys).unwrap();
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
assert_eq!(keys, deser);
}
#[test]
fn serde_ignores_unknown_fields() {
let json = r#"{"openai":"sk-x","unknown_field":"value","custom_endpoints":[]}"#;
let keys: ApiKeys = serde_json::from_str(json).unwrap();
assert_eq!(keys.openai, Some("sk-x".into()));
assert!(keys.custom_endpoints.is_empty());
}
// ── has_any_key ─────────────────────────────────────────────────
#[test]
fn has_any_key_false_when_empty() {
assert!(!ApiKeys::default().has_any_key());
}
#[test]
fn has_any_key_true_for_openai_only() {
let keys = ApiKeys {
openai: Some("sk-x".into()),
..Default::default()
};
assert!(keys.has_any_key());
}
#[test]
fn has_any_key_true_for_custom_endpoints_only() {
let keys = ApiKeys {
custom_endpoints: vec![endpoint("ep", "https://a.io", "key", &[("m", None)])],
..Default::default()
};
assert!(keys.has_any_key());
}
#[test]
fn has_any_key_false_for_endpoint_with_empty_api_key() {
let keys = ApiKeys {
custom_endpoints: vec![endpoint("ep", "https://a.io", "", &[("m", None)])],
..Default::default()
};
assert!(!keys.has_any_key());
}
// ── provider_key_count ─────────────────────────────────────────
#[test]
fn provider_key_count_zero_when_empty() {
assert_eq!(ApiKeys::default().provider_key_count(), 0);
}
#[test]
fn provider_key_count_counts_each_provider_key() {
let keys = ApiKeys {
openai: Some("sk-o".into()),
anthropic: Some("sk-a".into()),
google: Some("AIza".into()),
open_router: Some("sk-or".into()),
custom_endpoints: vec![],
};
assert_eq!(keys.provider_key_count(), 4);
}
#[test]
fn provider_key_count_ignores_blank_keys_and_endpoints() {
let keys = ApiKeys {
openai: Some("sk-o".into()),
anthropic: Some(" ".into()),
google: None,
open_router: None,
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
};
// Only the non-blank OpenAI key counts; the whitespace Anthropic key and the
// custom endpoint are excluded.
assert_eq!(keys.provider_key_count(), 1);
}
// ── custom_model_providers_for_request ──────────────────────────
#[test]
fn custom_model_providers_none_when_empty() {
let mgr = make_manager(ApiKeys::default());
assert!(mgr.custom_model_providers_for_request(true).is_none());
}
#[test]
fn custom_model_providers_none_when_byo_disabled() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
..Default::default()
});
assert!(mgr.custom_model_providers_for_request(false).is_none());
}
#[test]
fn custom_model_providers_populates_single_endpoint() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![endpoint_with_keys(
"My EP",
"https://custom.io/v1",
"ep-key",
&[("big-model", Some("alias"), "uuid-1")],
)],
..Default::default()
});
let result = mgr.custom_model_providers_for_request(true).unwrap();
assert_eq!(result.providers.len(), 1);
let p = &result.providers[0];
assert_eq!(p.base_url, "https://custom.io/v1");
assert_eq!(p.api_key, "ep-key");
assert_eq!(p.models.len(), 1);
assert_eq!(p.models[0].slug, "big-model");
assert_eq!(p.models[0].config_key, "uuid-1");
}
#[test]
fn multiple_endpoints_all_serialize() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![
endpoint_with_keys(
"ep1",
"https://a.io",
"k1",
&[("gpt-4", Some("fast"), "uuid-a")],
),
endpoint_with_keys(
"ep2",
"https://b.io",
"k2",
&[
("llama-70b", None, "uuid-b"),
("mixtral", Some("mix"), "uuid-c"),
],
),
],
..Default::default()
});
let result = mgr.custom_model_providers_for_request(true).unwrap();
assert_eq!(result.providers.len(), 2);
assert_eq!(result.providers[0].base_url, "https://a.io");
assert_eq!(result.providers[0].models[0].config_key, "uuid-a");
assert_eq!(result.providers[1].base_url, "https://b.io");
assert_eq!(result.providers[1].models.len(), 2);
assert_eq!(result.providers[1].models[0].slug, "llama-70b");
assert_eq!(result.providers[1].models[0].config_key, "uuid-b");
assert_eq!(result.providers[1].models[1].config_key, "uuid-c");
}
#[test]
fn byok_disabled_returns_none_even_with_endpoints() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
..Default::default()
});
assert!(mgr.custom_model_providers_for_request(false).is_none());
}
#[test]
fn empty_api_key_endpoints_are_skipped() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![
endpoint_with_keys("empty", "https://a.io", "", &[("m", None, "uuid-x")]),
endpoint_with_keys("ok", "https://b.io", "k", &[("m", None, "uuid-y")]),
],
..Default::default()
});
let result = mgr.custom_model_providers_for_request(true).unwrap();
assert_eq!(result.providers.len(), 1);
assert_eq!(result.providers[0].base_url, "https://b.io");
}
#[test]
fn endpoints_with_only_empty_models_are_skipped() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![endpoint_with_keys(
"ep",
"https://a.io",
"k",
&[("", None, "uuid-z")],
)],
..Default::default()
});
assert!(mgr.custom_model_providers_for_request(true).is_none());
}
// ── display_label fallback ─────────────────────────────────────
#[test]
fn display_label_uses_alias_when_present() {
let m = CustomEndpointModel {
name: "raw-name".into(),
alias: Some("My Alias".into()),
config_key: "k".into(),
};
assert_eq!(m.display_label(), "My Alias");
}
#[test]
fn display_label_falls_back_to_name_when_alias_missing() {
let m = CustomEndpointModel {
name: "raw-name".into(),
alias: None,
config_key: "k".into(),
};
assert_eq!(m.display_label(), "raw-name");
}
#[test]
fn display_label_falls_back_to_name_when_alias_is_whitespace() {
let m = CustomEndpointModel {
name: "raw-name".into(),
alias: Some(" ".into()),
config_key: "k".into(),
};
assert_eq!(m.display_label(), "raw-name");
}
// ── api_keys_for_request ────────────────────────────────────────
#[test]
fn api_keys_for_request_none_when_empty() {
let mgr = make_manager(ApiKeys::default());
assert!(mgr.api_keys_for_request(true, false, None).is_none());
}
#[test]
fn api_keys_for_request_populates_provider_keys() {
let mgr = make_manager(ApiKeys {
openai: Some("sk-o".into()),
anthropic: Some("sk-a".into()),
..Default::default()
});
let result = mgr.api_keys_for_request(true, false, None).unwrap();
assert_eq!(result.openai, "sk-o");
assert_eq!(result.anthropic, "sk-a");
assert!(result.google.is_empty());
}
#[test]
fn api_keys_for_request_omits_keys_when_byo_disabled() {
let mgr = make_manager(ApiKeys {
openai: Some("sk-o".into()),
..Default::default()
});
// With BYO disabled and no other credentials, returns None.
assert!(mgr.api_keys_for_request(false, false, None).is_none());
}
#[test]
fn api_keys_for_request_none_for_custom_endpoints_only() {
let mgr = make_manager(ApiKeys {
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
..Default::default()
});
assert!(mgr.api_keys_for_request(true, false, None).is_none());
}
// ── grok oauth token ────────────────────────────────────────────
#[test]
fn grok_access_token_present_without_expiry() {
let t = GrokTokens {
access_token: "tok".into(),
..Default::default()
};
assert_eq!(t.access_token_for_request(), Some("tok"));
}
#[test]
fn grok_access_token_blank_is_none() {
let t = GrokTokens {
access_token: " ".into(),
..Default::default()
};
assert_eq!(t.access_token_for_request(), None);
}
#[test]
fn grok_access_token_near_expiry_still_sent() {
// Expired tokens are still sent; the server is the authority on validity.
let t = grok_tokens("tok", Some(0));
assert_eq!(t.access_token_for_request(), Some("tok"));
}
#[test]
fn grok_access_token_far_future_is_some() {
let t = grok_tokens("tok", Some(3600));
assert_eq!(t.access_token_for_request(), Some("tok"));
}
#[test]
fn grok_needs_refresh_within_lead_time() {
assert!(grok_tokens("tok", Some(30)).needs_refresh(Duration::from_secs(300)));
assert!(!grok_tokens("tok", Some(3600)).needs_refresh(Duration::from_secs(300)));
// Expired tokens still need a refresh.
assert!(grok_tokens("tok", Some(0)).needs_refresh(Duration::from_secs(300)));
// Unknown expiry never reports as needing refresh.
assert!(!grok_tokens("tok", None).needs_refresh(Duration::from_secs(300)));
}
#[test]
fn api_keys_for_request_includes_grok_token() {
let mgr = make_manager_with_grok(
ApiKeys::default(),
Some(grok_tokens("grok-abc", Some(3600))),
);
let result = mgr.api_keys_for_request(true, false, None).unwrap();
assert_eq!(result.grok_oauth_access_token, "grok-abc");
assert!(result.anthropic.is_empty());
}
#[test]
fn api_keys_for_request_omits_grok_token_when_byo_disabled() {
// The Grok subscription is user-provided auth, so it follows the BYO
// policy gate: with BYO disabled and no other credentials, returns None.
let mgr = make_manager_with_grok(
ApiKeys::default(),
Some(grok_tokens("grok-abc", Some(3600))),
);
assert!(mgr.api_keys_for_request(false, false, None).is_none());
}
#[test]
fn api_keys_for_request_includes_expired_grok_token() {
// Expired tokens are still sent in requests; the server rejects truly
// invalid ones and the background refresh replaces them.
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens("grok-abc", Some(0))));
let result = mgr.api_keys_for_request(true, false, None).unwrap();
assert_eq!(result.grok_oauth_access_token, "grok-abc");
}
#[test]
fn has_grok_subscription_false_when_not_connected() {
let mgr = make_manager(ApiKeys::default());
assert!(!mgr.has_grok_subscription());
}
#[test]
fn has_grok_subscription_true_when_connected() {
let mgr = make_manager_with_grok(
ApiKeys::default(),
Some(grok_tokens("grok-abc", Some(3600))),
);
assert!(mgr.has_grok_subscription());
}
#[test]
fn has_grok_subscription_true_for_expired_token() {
// A connected subscription still counts even when its token is past expiry:
// the token is sent anyway and the server is the authority on validity.
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens("grok-abc", Some(0))));
assert!(mgr.has_grok_subscription());
}
#[test]
fn has_grok_subscription_false_when_token_blank() {
// A blank token can't be sent, so it does not count as a usable credential.
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens(" ", None)));
assert!(!mgr.has_grok_subscription());
}
// ── ApiKeyManager::has_any_key ──────────────────
#[test]
fn manager_has_any_key_false_when_no_keys_and_no_grok() {
let mgr = make_manager(ApiKeys::default());
assert!(!mgr.has_any_key());
}
#[test]
fn manager_has_any_key_true_for_pasted_key_without_grok() {
let mgr = make_manager(ApiKeys {
openai: Some("sk-x".into()),
..Default::default()
});
assert!(mgr.has_any_key());
}
#[test]
fn manager_has_any_key_true_for_connected_grok_without_pasted_key() {
// The crux: a connected Grok subscription counts even with no pasted keys,
// matching how it's sent as a BYO credential on requests.
let mgr = make_manager_with_grok(
ApiKeys::default(),
Some(grok_tokens("grok-abc", Some(3600))),
);
assert!(mgr.has_any_key());
}
#[test]
fn manager_has_any_key_false_for_blank_grok_and_no_keys() {
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens(" ", None)));
assert!(!mgr.has_any_key());
}
// ── geap credentials ────────────────────────────────────────────
#[test]
fn geap_access_token_present_without_expiry() {
let credentials = GeapCredentials::new("tok".into(), None);
assert_eq!(credentials.access_token_for_request(), Some("tok"));
}
#[test]
fn geap_access_token_blank_is_none() {
let credentials = GeapCredentials::new(" ".into(), None);
assert_eq!(credentials.access_token_for_request(), None);
}
#[test]
fn geap_access_token_near_expiry_still_sent() {
// Expired tokens are still sent; Google is the authority on validity.
let credentials = geap_credentials("tok", Some(0));
assert_eq!(credentials.access_token_for_request(), Some("tok"));
}
#[test]
fn geap_needs_refresh_lead_time_boundaries() {
// Within the 5-minute lead window.
assert!(geap_credentials("tok", Some(30)).needs_refresh());
// Comfortably fresh.
assert!(!geap_credentials("tok", Some(3600)).needs_refresh());
// Already expired -> still needs a refresh.
assert!(geap_credentials("tok", Some(0)).needs_refresh());
// Unknown expiry never reports as needing a refresh.
assert!(!geap_credentials("tok", None).needs_refresh());
}
#[test]
fn api_keys_for_request_includes_geap_token_when_gate_and_binding_match() {
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
let result = mgr
.api_keys_for_request(false, false, Some(geap_gate()))
.unwrap();
let credentials = result.google_cloud_credentials.unwrap();
assert_eq!(credentials.access_token, "geap-abc");
// The GEAP token is independent of the BYO key gate.
assert!(result.anthropic.is_empty());
}
#[test]
fn api_keys_for_request_includes_expired_geap_token() {
// Expired tokens are still attached — never silently dropped. Google
// rejects truly invalid ones, which surfaces a recoverable error instead
// of a silent fallback to another route.
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(0)));
let result = mgr
.api_keys_for_request(false, false, Some(geap_gate()))
.unwrap();
assert_eq!(
result.google_cloud_credentials.unwrap().access_token,
"geap-abc"
);
}
#[test]
fn api_keys_for_request_omits_geap_token_without_gate() {
// No gate (policy off at the call site) ⇒ no GEAP credentials, even when
// a token is loaded.
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
assert!(mgr.api_keys_for_request(false, false, None).is_none());
}
#[test]
fn api_keys_for_request_omits_geap_token_on_binding_mismatch() {
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
// A different user (sign-out/account switch).
let mut gate = geap_gate();
gate.user_uid = "someone-else".into();
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
// A different audience (admin changed the pool/provider).
let mut gate = geap_gate();
gate.audience = "//iam.googleapis.com/projects/2/locations/global/workloadIdentityPools/other/providers/other".into();
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
// A different service account (admin changed impersonation target).
let mut gate = geap_gate();
gate.federation = GeapFederation::ServiceAccount {
email: "other@proj.iam.gserviceaccount.com".into(),
};
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
}
#[test]
fn api_keys_for_request_serves_previous_geap_token_while_refreshing() {
// A re-mint in flight keeps serving the previous token — tokens stay
// until replaced.
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing {
previous: Some((geap_credentials("geap-old", Some(10)), geap_binding())),
});
let result = mgr
.api_keys_for_request(false, false, Some(geap_gate()))
.unwrap();
assert_eq!(
result.google_cloud_credentials.unwrap().access_token,
"geap-old"
);
}
#[test]
fn api_keys_for_request_omits_geap_token_during_first_mint() {
// The very first mint has nothing to serve yet.
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing { previous: None });
assert!(mgr
.api_keys_for_request(false, false, Some(geap_gate()))
.is_none());
}
#[test]
fn api_keys_for_request_omits_geap_token_for_non_loaded_states() {
for state in [
GeapCredentialsState::Missing,
GeapCredentialsState::Disabled,
GeapCredentialsState::Failed {
error: LoadGeapCredentialsError::ExchangeToken {
status: None,
detail: "boom".into(),
},
},
] {
let mgr = make_manager_with_geap(state);
assert!(mgr
.api_keys_for_request(false, false, Some(geap_gate()))
.is_none());
}
}
#[test]
fn api_keys_for_request_omits_geap_token_when_previous_binding_mismatches() {
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing {
previous: Some((geap_credentials("geap-old", Some(10)), geap_binding())),
});
let mut gate = geap_gate();
gate.user_uid = "someone-else".into();
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
}
+15 -1
View File
@@ -6,7 +6,7 @@ use warp_multi_agent_api as api;
/// Temporary AWS credentials loaded from the AWS SDK.
/// These are not persisted and are only used at runtime.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Clone, PartialEq, Eq)]
pub struct AwsCredentials {
access_key: String,
secret_key: String,
@@ -14,6 +14,20 @@ pub struct AwsCredentials {
expires_at: Option<SystemTime>,
}
impl std::fmt::Debug for AwsCredentials {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AwsCredentials")
.field("access_key", &"<redacted>")
.field("secret_key", &"<redacted>")
.field(
"session_token",
&self.session_token.as_ref().map(|_| "<redacted>"),
)
.field("expires_at", &self.expires_at)
.finish()
}
}
impl AwsCredentials {
pub fn new(
access_key: String,
+77 -15
View File
@@ -1,14 +1,13 @@
use std::cmp::Ordering;
use std::fmt::{self, Display};
use std::ops::Range;
use std::path::PathBuf;
use std::sync::LazyLock;
use itertools::{EitherOrBoth, Itertools};
use lazy_static::lazy_static;
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::{
cmp::Ordering,
fmt::{self, Display},
ops::Range,
path::PathBuf,
sync::LazyLock,
};
use strsim::jaro_winkler;
lazy_static! {
/// Regex to parse a line number from a string in the format "{number}|{line}"
@@ -283,6 +282,35 @@ fn unmatched_line_suffix<'a>(search_line: &str, file_line: &'a str) -> Option<&'
}
}
/// Preserve the unmatched suffix of the final matched file line when the LLM emitted only a
/// partial final line.
///
/// When search and replace have the same line count, preserve the suffix to maintain the legacy
/// behavior for simple partial-line replacements. When line counts differ, only preserve it if the
/// replacement's final line still carries the same partial context as the search's final line.
fn append_unmatched_line_suffix(search: &str, file_line: &str, insertion: &mut String) {
let Some(search_last_line) = lines(search).last() else {
return;
};
let Some(suffix) = unmatched_line_suffix(search_last_line, file_line) else {
return;
};
let search_line_count = lines(search).count();
let insertion_line_count = lines(insertion).count();
let Some(insertion_last_line) = lines(insertion).last() else {
return;
};
if search_line_count != insertion_line_count
&& search_last_line.trim_start() != insertion_last_line.trim_start()
{
return;
}
let insertion_point = insertion.trim_end_matches('\n').len();
insertion.insert_str(insertion_point, suffix);
}
/// We told the model not to include line numbers for the replacement content. However, it can
/// still happen. Try to remove them here.
/// https://github.com/warpdotdev/warp-server/blob/d9c1b6d1443290f2355979ae552d41af01a63bde/logic/ai/prompt/tools/suggest_diff.yaml#L34-L34
@@ -402,6 +430,14 @@ pub fn fuzzy_match_v4a_diffs(
}
}
// Sort by start line and remove overlapping deltas. When the LLM produces
// multiple hunks targeting the same region (e.g. a large deletion whose
// matched range subsumes a nearby single-line edit), the overlapping delta
// must be dropped — applying both would produce an invalid edit range in
// the editor buffer (see WARP-CLIENT-DEV-NYY).
deltas.sort_by_key(|d| d.replacement_line_range.start);
deltas = deduplicate_overlapping_deltas(deltas);
let update_deltas_empty = deltas.is_empty();
let failures = if failures.fuzzy_match_failures > 0
|| failures.missing_line_numbers > 0
@@ -420,6 +456,33 @@ pub fn fuzzy_match_v4a_diffs(
}
}
/// Given a list of `DiffDelta`s sorted by `replacement_line_range.start`,
/// drop any delta whose range overlaps with the preceding accepted delta.
///
/// "Overlaps" means `B.start < A.end` (strictly inside or partial overlap).
/// Adjacent ranges (`A.end == B.start`) are kept.
fn deduplicate_overlapping_deltas(sorted_deltas: Vec<DiffDelta>) -> Vec<DiffDelta> {
let mut result: Vec<DiffDelta> = Vec::with_capacity(sorted_deltas.len());
for delta in sorted_deltas {
let dominated = result.last().is_some_and(|prev| {
delta.replacement_line_range.start < prev.replacement_line_range.end
});
if dominated {
log::warn!(
"Dropping V4A delta with overlapping range {:?} \
(subsumed by preceding delta with range {:?})",
delta.replacement_line_range,
result.last().unwrap().replacement_line_range,
);
continue;
}
result.push(delta);
}
result
}
fn fuzzy_match_file_diffs(
diffs: &[SearchAndReplace],
file_content: &str,
@@ -547,13 +610,12 @@ fn fuzzy_match_file_diffs(
// the delta would replace the entire line and drop the unmatched
// suffix. Detect this and preserve the suffix in the insertion.
let mut insertion = diff.replace.clone();
if range.end >= 2 && lines(&search).count() == lines(&insertion).count() {
if let Some(suffix) = lines(&search)
.last()
.and_then(|last| unmatched_line_suffix(last, target_lines[range.end - 2]))
{
insertion.push_str(suffix);
}
if range.end >= 2 {
append_unmatched_line_suffix(
&search,
target_lines[range.end - 2],
&mut insertion,
);
}
deltas.push(DiffDelta {
replacement_line_range: range.start..range.end,
@@ -1208,5 +1270,5 @@ fn find_change_context_start(change_context: &[String], file_lines: &[&str]) ->
}
#[cfg(test)]
#[path = "mod_test.rs"]
#[path = "mod_tests.rs"]
mod tests;
@@ -608,6 +608,66 @@ fn test_partial_last_line_in_search_preserves_suffix() {
assert_eq!(result, "func foo() {\nlet y = 1;\nlet x = 2;\n}\n");
}
#[test]
fn test_partial_last_line_in_multiline_replacement_preserves_suffix() {
// This mirrors a model edit that deletes middle lines while leaving the final line as partial
// trailing context. The final line should remain a no-op after suffix preservation.
let file_content = "\
mod proxy;
pub fn run_daemon() -> anyhow::Result<()> {
// Logging is now handled by init_common (log_destination: File).
// socket_path: ~/.warp[-channel]/remote-server/server.sock
// The Unix domain socket the daemon binds on.
}
";
let diffs = [SearchAndReplace {
search: "\
2|pub fn run_daemon() -> anyhow::Result<()> {
3| // Logging is now handled by init_common (log_destination: File).
4|
5| // socket_path:"
.to_string(),
replace: "\
pub fn run_daemon() -> anyhow::Result<()> {
// socket_path:"
.to_string(),
}];
let (deltas, _failures) = fuzzy_match_file_diffs(&diffs, file_content);
assert_eq!(deltas.len(), 1, "Expected one matched delta");
assert_eq!(deltas[0].replacement_line_range, 2..6);
assert_eq!(
deltas[0].insertion,
"pub fn run_daemon() -> anyhow::Result<()> {\n // socket_path: ~/.warp[-channel]/remote-server/server.sock"
);
let file_lines: Vec<&str> = file_content.lines().collect();
let range = &deltas[0].replacement_line_range;
let mut result = String::new();
for line in &file_lines[..range.start - 1] {
result.push_str(line);
result.push('\n');
}
result.push_str(&deltas[0].insertion);
result.push('\n');
for line in &file_lines[range.end - 1..] {
result.push_str(line);
result.push('\n');
}
assert_eq!(
result,
"\
mod proxy;
pub fn run_daemon() -> anyhow::Result<()> {
// socket_path: ~/.warp[-channel]/remote-server/server.sock
// The Unix domain socket the daemon binds on.
}
"
);
}
#[test]
fn test_search_and_replace_accommodates_none() {
let parsed_diff = ParsedDiff::StrReplaceEdit {
@@ -679,3 +739,88 @@ fn test_custom_lines() {
assert_eq!(lines("foo\nbar").collect_vec(), vec!["foo", "bar"]);
assert_eq!(lines("foo\nbar\n").collect_vec(), vec!["foo", "bar"]);
}
/// Regression test for WARP-CLIENT-DEV-NYY: panic "Invalid edit range 4042..3982".
///
/// Reproduces the crash from MAA conversation d71bf84b (request b621adb3).
/// Two V4A hunks target the same region: a large deletion whose matched range
/// subsumes a nearby single-line edit. Without `deduplicate_overlapping_deltas`,
/// both deltas survive and `Buffer::edit` panics on the overlapping ranges.
#[test]
fn test_v4a_maa_crash_d71bf84b_no_overlapping_deltas() {
// File content where hunk A (deletion) and hunk B (delegate tweak) both
// match, and hunk A's matched range fully contains hunk B's.
// The `ActiveMicButtonTheme.background` line that hunk B targets sits
// inside `DefaultWeightAgentInputButtonTheme`'s impl, so hunk A's
// deletion (which covers the whole impl) subsumes hunk B.
let file_content = "\
}\n\
}\n\
}\n\
\n\
struct DefaultWeightAgentInputButtonTheme;\n\
\n\
impl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n\
fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {\n\
AgentInputButtonTheme.background(hovered, appearance)\n\
}\n\
\n\
fn text_color(\n\
&self,\n\
hovered: bool,\n\
background: Option<Fill>,\n\
appearance: &Appearance,\n\
) -> ColorU {\n\
AgentInputButtonTheme.text_color(hovered, background, appearance)\n\
}\n\
\n\
fn border(&self, appearance: &Appearance) -> Option<ColorU> {\n\
AgentInputButtonTheme.border(appearance)\n\
}\n\
\n\
fn should_opt_out_of_contrast_adjustment(&self) -> bool {\n\
true\n\
}\n\
}";
let hunks = vec![
// Hunk A: delete the entire DefaultWeightAgentInputButtonTheme block.
V4AHunk {
change_context: vec![],
pre_context: " }\n }\n}".to_string(),
old: "\nstruct DefaultWeightAgentInputButtonTheme;\n\nimpl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {\n AgentInputButtonTheme.background(hovered, appearance)\n }\n\n fn text_color(\n &self,\n hovered: bool,\n background: Option<Fill>,\n appearance: &Appearance,\n ) -> ColorU {\n AgentInputButtonTheme.text_color(hovered, background, appearance)\n }\n\n fn border(&self, appearance: &Appearance) -> Option<ColorU> {\n AgentInputButtonTheme.border(appearance)\n }\n\n fn should_opt_out_of_contrast_adjustment(&self) -> bool {\n true\n }\n}".to_string(),
new: String::new(),
post_context: String::new(),
},
// Hunk B: tweak a delegate call inside the same region hunk A deletes.
// Its preContext + old match a line inside hunk A's range, so it
// produces a delta whose range overlaps with hunk A's.
V4AHunk {
change_context: vec![],
pre_context: "impl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {".to_string(),
old: " AgentInputButtonTheme.background(hovered, appearance)".to_string(),
new: " AgentInputButtonTheme::default().background(hovered, appearance)".to_string(),
post_context: " }".to_string(),
},
];
let diff = fuzzy_match_v4a_diffs("mod.rs", &hunks, None, file_content);
let deltas = deltas(&diff);
// Hunk B's matched range is inside hunk A's, so deduplication must drop it.
// Only hunk A's delta (the deletion) should survive.
assert_eq!(
deltas.len(),
1,
"Expected 1 delta (subsumed hunk should be dropped), got {}: {:?}",
deltas.len(),
deltas
.iter()
.map(|d| &d.replacement_line_range)
.collect::<Vec<_>>(),
);
assert!(
deltas[0].insertion.is_empty(),
"The surviving delta should be the deletion"
);
}
+91
View File
@@ -0,0 +1,91 @@
use std::time::{Duration, SystemTime};
use warp_multi_agent_api as api;
/// Refresh the access token this long before its hard expiry
pub const GEAP_REFRESH_LEAD_TIME: Duration = Duration::from_secs(5 * 60);
#[derive(Clone, PartialEq, Eq)]
pub struct GeapCredentials {
access_token: String,
expires_at: Option<SystemTime>,
}
impl std::fmt::Debug for GeapCredentials {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GeapCredentials")
.field("access_token", &"<redacted>")
.field("expires_at", &self.expires_at)
.finish()
}
}
impl GeapCredentials {
pub fn new(access_token: String, expires_at: Option<SystemTime>) -> Self {
Self {
access_token,
expires_at,
}
}
pub fn expires_at(&self) -> Option<SystemTime> {
self.expires_at
}
pub fn access_token_for_request(&self) -> Option<&str> {
(!self.access_token.trim().is_empty()).then_some(self.access_token.as_str())
}
pub fn needs_refresh(&self) -> bool {
match self.expires_at {
Some(expires_at) => expires_at <= SystemTime::now() + GEAP_REFRESH_LEAD_TIME,
None => false,
}
}
}
impl From<GeapCredentials> for api::request::settings::api_keys::GoogleCloudCredentials {
fn from(credentials: GeapCredentials) -> Self {
Self {
access_token: credentials.access_token,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GeapFederation {
DirectWif,
ServiceAccount { email: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GeapMintBinding {
pub user_uid: String,
pub audience: String,
pub federation: GeapFederation,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LoadGeapCredentialsError {
MintIdentityToken { detail: String },
ExchangeToken { status: Option<u16>, detail: String },
ImpersonateServiceAccount { status: Option<u16>, detail: String },
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum GeapCredentialsState {
#[default]
Missing,
Disabled,
Refreshing {
previous: Option<(GeapCredentials, GeapMintBinding)>,
},
Loaded {
credentials: GeapCredentials,
loaded_at: SystemTime,
minted_for: GeapMintBinding,
},
Failed {
error: LoadGeapCredentialsError,
},
}
+228
View File
@@ -0,0 +1,228 @@
//! Refresh orchestration for a connected xAI / Grok subscription's OAuth
//! tokens.
//!
//! The tokens themselves live in [`ApiKeyManager`] (the request-building
//! source of truth, persisted to secure storage under `GrokOAuthTokens`).
//! This module owns the network-facing refresh lifecycle — converting a
//! [`TokenResponse`] into stored [`GrokTokens`], proactively refreshing the
//! access token shortly before it expires, and rescheduling the next refresh.
//!
//! The Grok subscription is BYO auth, so background refresh follows the BYO
//! API key policy. That policy lives in the app layer (workspace settings),
//! which this crate has no visibility into; the app wires it in via
//! [`ApiKeyManager::set_grok_refresh_allowed`].
//!
//! The network/protocol side of the connect flow (authorize URL, loopback
//! callback server, token exchange/refresh) lives in the [`oauth`] submodule.
pub mod oauth;
use std::time::{Duration, SystemTime};
use galaxyui_core::r#async::Timer;
use galaxyui_core::ModelContext;
use self::oauth::TokenResponse;
use crate::api_keys::{ApiKeyManager, GrokTokens};
/// Refresh the access token this long before its hard expiry so a request
/// never races the expiration. Possibly-expired tokens are still sent (the
/// server is the authority on validity), so this lead time is purely about
/// keeping the token fresh, not about when it stops being sent.
const REFRESH_LEAD_TIME: Duration = Duration::from_secs(5 * 60);
/// Builds [`GrokTokens`] from a token-endpoint [`TokenResponse`], computing the
/// absolute `expires_at` from the relative `expires_in`. Values not present in
/// the response are carried over from `previous`: the refresh token when xAI
/// doesn't return a new one (refresh-token rotation is optional in OAuth 2.0),
/// and `connected_at` so it keeps reflecting the initial connection time
/// (initialized to now when there are no previous tokens, i.e. a fresh
/// connect).
pub fn grok_tokens_from_response(
response: TokenResponse,
previous: Option<&GrokTokens>,
) -> GrokTokens {
let expires_at = response
.expires_in
.and_then(|secs| u64::try_from(secs).ok())
.and_then(|secs| SystemTime::now().checked_add(Duration::from_secs(secs)));
GrokTokens {
access_token: response.access_token,
refresh_token: response
.refresh_token
.or_else(|| previous.and_then(|tokens| tokens.refresh_token.clone())),
expires_at,
connected_at: previous
.and_then(|tokens| tokens.connected_at)
.or_else(|| Some(SystemTime::now())),
}
}
impl ApiKeyManager {
/// Persists freshly obtained tokens (e.g. right after the connect flow) and
/// schedules the next proactive refresh.
pub fn store_grok_tokens(&mut self, response: TokenResponse, ctx: &mut ModelContext<Self>) {
apply_grok_tokens(self, response, ctx);
}
/// Updates whether background refresh of the stored Grok tokens is
/// allowed. The Grok subscription is BYO auth, so refresh follows the same
/// policy gate as request injection ([`Self::api_keys_for_request`]):
/// tokens that can never be sent shouldn't be kept fresh. The policy lives
/// in the app layer, which calls this at startup and whenever the policy
/// may have changed (e.g. team data arriving, or a workspace switch).
///
/// Schedules a refresh on a disabled -> enabled transition (refreshing
/// immediately if the token has already (nearly) expired); in-flight
/// timers re-check the flag when they fire. Repeated calls with an
/// unchanged value are no-ops, so duplicate timers can't pile up.
pub fn set_grok_refresh_allowed(&mut self, allowed: bool, ctx: &mut ModelContext<Self>) {
if self.grok_refresh_allowed == allowed {
return;
}
self.grok_refresh_allowed = allowed;
if allowed {
schedule_grok_token_refresh(self, ctx);
}
}
/// Request-time safety net: kicks off a background refresh of the stored
/// Grok tokens when they are nearing (or already past) expiry, so
/// upcoming requests can authenticate even if the proactive refresh loop
/// never armed or died (e.g. a stale BYO policy at startup, or an earlier
/// failed refresh). The triggering request still carries the currently
/// stored token — the server is the authority on its validity.
///
/// `byo_allowed` is the BYO API key policy as freshly evaluated by the
/// caller at request time. It also re-syncs the stored policy mirror,
/// which can go stale between `TeamsChanged` events; a disabled ->
/// enabled transition re-arms the proactive refresh loop.
pub fn refresh_grok_tokens_if_needed(
&mut self,
byo_allowed: bool,
ctx: &mut ModelContext<Self>,
) {
self.set_grok_refresh_allowed(byo_allowed, ctx);
if !byo_allowed || self.grok_refresh_in_flight {
return;
}
let Some(tokens) = self.grok_tokens() else {
return;
};
if !tokens.needs_refresh(REFRESH_LEAD_TIME) {
return;
}
let Some(refresh_token) = tokens.refresh_token.clone() else {
return;
};
log::info!(
"Grok OAuth token is nearing or past expiry at request time; refreshing in background"
);
spawn_grok_refresh(self, refresh_token, ctx);
}
}
/// Stores the tokens from `response` (carrying over the previous refresh token
/// and connection time when absent) and schedules the next proactive refresh.
fn apply_grok_tokens(
manager: &mut ApiKeyManager,
response: TokenResponse,
ctx: &mut ModelContext<ApiKeyManager>,
) {
let tokens = grok_tokens_from_response(response, manager.grok_tokens());
manager.set_grok_tokens(Some(tokens), ctx);
schedule_grok_token_refresh(manager, ctx);
}
/// Schedules a one-shot proactive refresh [`REFRESH_LEAD_TIME`] before the
/// current token's expiry (immediately if already within that window).
///
/// No-op when there's nothing to refresh against (no tokens, no refresh token,
/// or no known expiry). Reschedules itself after each successful refresh, so a
/// single call establishes an ongoing refresh loop for the lifetime of the
/// connection.
fn schedule_grok_token_refresh(manager: &mut ApiKeyManager, ctx: &mut ModelContext<ApiKeyManager>) {
// When the BYO API key policy is disabled the token is never sent, so
// don't refresh it in the background either. `set_grok_refresh_allowed`
// re-establishes the loop if the policy is later enabled.
if !manager.grok_refresh_allowed {
return;
}
let Some(tokens) = manager.grok_tokens() else {
return;
};
let Some(refresh_token) = tokens.refresh_token.clone() else {
return;
};
let Some(expires_at) = tokens.expires_at else {
// No expiry signal, so there's nothing to schedule against.
return;
};
let now = SystemTime::now();
let fire_at = expires_at.checked_sub(REFRESH_LEAD_TIME).unwrap_or(now);
let delay = fire_at.duration_since(now).unwrap_or(Duration::ZERO);
ctx.spawn(
async move {
Timer::after(delay).await;
},
move |manager, _output, ctx| {
// The BYO policy may have flipped off while we slept;
// `set_grok_refresh_allowed` restarts the loop if it flips back
// on.
if !manager.grok_refresh_allowed {
return;
}
// The stored token may have changed (reconnect/disconnect) while we
// slept; only refresh if our refresh token is still the current one.
let still_current = manager
.grok_tokens()
.and_then(|t| t.refresh_token.as_deref())
== Some(refresh_token.as_str());
if still_current {
spawn_grok_refresh(manager, refresh_token, ctx);
}
},
);
}
/// Kicks off a background token refresh using `refresh_token`, applying the
/// result (which reschedules the next refresh) or logging the failure.
///
/// No-op when a refresh is already in flight, so the proactive timer and the
/// request-time safety net can't issue overlapping refreshes.
fn spawn_grok_refresh(
manager: &mut ApiKeyManager,
refresh_token: String,
ctx: &mut ModelContext<ApiKeyManager>,
) {
if manager.grok_refresh_in_flight {
return;
}
manager.grok_refresh_in_flight = true;
ctx.spawn(
async move { oauth::refresh_access_token(&refresh_token).await },
|manager, result, ctx| {
manager.grok_refresh_in_flight = false;
match result {
Ok(response) => {
log::info!(
"Refreshed Grok OAuth token (expires_in={:?}, has_refresh_token={})",
response.expires_in,
response.refresh_token.is_some(),
);
apply_grok_tokens(manager, response, ctx);
}
Err(err) => {
// Leave the existing (possibly expired) token in place; the
// server remains the authority and will reject it if it's
// truly invalid. The request-time safety net
// (`ApiKeyManager::refresh_grok_tokens_if_needed`) retries
// on the next request.
log::error!("Failed to refresh Grok OAuth token: {err:#}");
}
}
},
);
}
+457
View File
@@ -0,0 +1,457 @@
//! OAuth flow for connecting an xAI / Grok subscription (e.g. SuperGrok) to
//! Warp, so users can "plug in" their subscription instead of pasting a
//! pay-as-you-go API key.
//!
//! This mirrors the public Grok-CLI desktop OAuth flow: an OAuth 2.0
//! Authorization Code grant with PKCE and a fixed loopback redirect URI. xAI's
//! auth server only accepts the loopback redirect for an allowlisted
//! `client_id` bound to a specific port, so we reuse the Grok-CLI client and
//! bind the callback server to that exact port.
//!
//! Some browsers/networks can't reach the loopback callback (e.g. Private
//! Network Access is blocked), in which case xAI's consent screen instead
//! *displays* the authorization code for the user to paste back into the app.
//! [`OauthAttempt::manual_code_exchange`] supports that fallback by capturing
//! the attempt's PKCE verifier so a pasted code can be exchanged directly,
//! without ever observing the loopback redirect.
//!
//! This module owns only the network/protocol side: building the authorize
//! URL, running the loopback callback server, and exchanging/refreshing tokens
//! at xAI's token endpoint. Persistence of the resulting tokens, proactive
//! refresh scheduling, and injection into the request live in the parent
//! [`crate::grok_subscription`] module (refresh orchestration) and
//! [`crate::api_keys::ApiKeyManager`] (storage + request injection).
use std::io::{ErrorKind, Read, Write};
use std::net::{Shutdown, TcpListener, TcpStream};
use std::time::Duration;
use anyhow::{bail, Context as _};
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64::Engine as _;
// `std::time::Instant` is disallowed (no wasm support); `instant::Instant` is a
// drop-in that re-exports the std type on native targets.
use instant::Instant;
use rand::RngCore as _;
use serde::Deserialize;
use sha2::{Digest, Sha256};
const CLIENT_ID: &str = "b1a00492-073a-47ea-816f-4c329264a828";
const AUTHORIZE_URL: &str = "https://auth.x.ai/oauth2/authorize";
const TOKEN_URL: &str = "https://auth.x.ai/oauth2/token";
const SCOPE: &str = "openid profile email offline_access grok-cli:access api:access";
const REDIRECT_HOST: &str = "127.0.0.1";
const REDIRECT_PORT: u16 = 56121;
/// How long we keep the loopback server open waiting for the user to approve
/// the consent screen in their browser.
const CALLBACK_TIMEOUT: Duration = Duration::from_secs(300);
/// How long to nap between non-blocking `accept()` attempts.
const POLL_INTERVAL: Duration = Duration::from_millis(100);
/// xAI's browser consent screen fetches the loopback callback from these
/// origins. Since that request crosses origins (https://accounts.x.ai ->
/// http://127.0.0.1), browsers require CORS and Private Network Access headers
/// before the page can observe the callback response.
const CORS_ALLOWED_ORIGINS: [&str; 2] = ["https://accounts.x.ai", "https://auth.x.ai"];
fn redirect_uri() -> String {
format!("http://{REDIRECT_HOST}:{REDIRECT_PORT}/callback")
}
/// One in-flight OAuth login attempt: the bound loopback callback listener
/// plus the per-attempt PKCE/CSRF secrets, which never leave this module.
///
/// Construct with [`OauthAttempt::start`], open [`OauthAttempt::authorize_url`]
/// in the browser, then await [`OauthAttempt::finish`] to obtain tokens. Tying
/// the secrets to the attempt guarantees the same PKCE verifier and CSRF state
/// are used for both the authorize URL and the code exchange.
pub struct OauthAttempt {
listener: TcpListener,
pkce: PkceParams,
}
impl OauthAttempt {
/// Binds the loopback callback server and generates fresh per-attempt
/// secrets. Call this before opening the browser so a bind failure (e.g.
/// another login already in progress, or Grok-CLI holding the port)
/// surfaces before a browser tab opens.
pub fn start() -> anyhow::Result<Self> {
Ok(Self {
listener: bind_callback_listener()?,
pkce: PkceParams::generate(),
})
}
/// The authorization URL the user's browser should open to begin the flow.
pub fn authorize_url(&self) -> String {
authorize_url(&self.pkce)
}
/// Runs the rest of the browser-based PKCE flow: waits for the loopback
/// callback, validates the CSRF state, and exchanges the authorization
/// code for tokens. Consumes the attempt so its secrets can't be reused.
pub async fn finish(self) -> anyhow::Result<TokenResponse> {
run_oauth_flow(self.listener, self.pkce).await
}
/// Clones the PKCE verifier for the pasted-code fallback while the
/// loopback flow continues racing in parallel.
pub fn manual_code_exchange(&self) -> ManualCodeExchange {
ManualCodeExchange {
verifier: self.pkce.verifier.clone(),
}
}
}
/// Completes OAuth from a manually-pasted authorization code.
///
/// There is no redirect `state` to validate in this out-of-band path; PKCE
/// protects the exchange.
#[derive(Clone)]
pub struct ManualCodeExchange {
verifier: String,
}
impl ManualCodeExchange {
/// Exchanges a user-pasted authorization `code` with the attempt's PKCE verifier.
pub async fn exchange(&self, code: &str) -> anyhow::Result<TokenResponse> {
let code = code.trim();
if code.is_empty() {
bail!("enter the code shown in your browser to finish connecting");
}
exchange_code_for_tokens(code, &self.verifier).await
}
}
/// The per-attempt secrets for one authorization request: the PKCE
/// verifier/challenge pair and the CSRF `state` value.
struct PkceParams {
verifier: String,
challenge: String,
/// CSRF token echoed back on the redirect and validated against the
/// response before the code is exchanged.
state: String,
}
impl PkceParams {
/// Generates a fresh PKCE verifier + S256 challenge and a random CSRF state.
fn generate() -> Self {
let verifier = random_url_safe_token();
let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()));
let state = random_url_safe_token();
Self {
verifier,
challenge,
state,
}
}
}
/// Returns a URL-safe, unpadded base64 string of 32 random bytes. This is used
/// for both the PKCE code verifier (RFC 7636 allows 43-128 chars from the
/// unreserved set) and the CSRF state.
fn random_url_safe_token() -> String {
let mut bytes = [0u8; 32];
rand::thread_rng().fill_bytes(&mut bytes);
URL_SAFE_NO_PAD.encode(bytes)
}
/// Builds the authorization URL the user's browser should open to begin the
/// flow.
fn authorize_url(pkce: &PkceParams) -> String {
let redirect = redirect_uri();
// `plan=generic` opts the consent screen into xAI's generic OAuth plan tier
// (required for loopback OAuth from non-allowlisted clients); `referrer`
// is best-effort attribution in xAI's OAuth logs.
let params: [(&str, &str); 9] = [
("response_type", "code"),
("client_id", CLIENT_ID),
("redirect_uri", &redirect),
("scope", SCOPE),
("code_challenge", &pkce.challenge),
("code_challenge_method", "S256"),
("state", &pkce.state),
("plan", "generic"),
("referrer", "warp"),
];
let query =
serde_urlencoded::to_string(params).expect("static OAuth params are always serializable");
format!("{AUTHORIZE_URL}?{query}")
}
/// The token endpoint's response. Fields beyond `access_token` are optional
/// because xAI does not always return them. Other response fields (e.g.
/// `token_type`, `scope`) are ignored since nothing consumes them.
#[derive(Debug, Deserialize)]
pub struct TokenResponse {
pub access_token: String,
#[serde(default)]
pub refresh_token: Option<String>,
#[serde(default)]
pub expires_in: Option<i64>,
}
/// The authorization code and state captured from the loopback redirect.
struct CallbackData {
code: String,
state: String,
}
/// Binds the loopback callback server to the fixed redirect address.
fn bind_callback_listener() -> anyhow::Result<TcpListener> {
let listener = TcpListener::bind((REDIRECT_HOST, REDIRECT_PORT)).with_context(|| {
format!(
"couldn't bind the Grok OAuth callback server to {REDIRECT_HOST}:{REDIRECT_PORT}. \
Another login may be in progress, or another app (e.g. Grok CLI) is using the port."
)
})?;
listener
.set_nonblocking(true)
.context("failed to set the Grok OAuth callback listener to non-blocking mode")?;
Ok(listener)
}
/// Runs the full browser-based PKCE flow: waits for the loopback callback on a
/// dedicated thread, validates the CSRF state, and exchanges the authorization
/// code for tokens.
async fn run_oauth_flow(listener: TcpListener, pkce: PkceParams) -> anyhow::Result<TokenResponse> {
// The loopback accept loop is blocking, so run it on a dedicated OS thread
// and bridge the result back through a runtime-agnostic async channel.
let (tx, rx) = async_channel::bounded(1);
std::thread::Builder::new()
.name("grok-oauth-callback".to_owned())
.spawn(move || {
// `send_blocking` is disallowed (no wasm support); block this
// dedicated thread on the async `send` instead.
let _ = galaxyui_core::r#async::block_on(
tx.send(wait_for_callback(&listener, CALLBACK_TIMEOUT)),
);
})
.context("failed to spawn the Grok OAuth callback server thread")?;
let callback = rx
.recv()
.await
.context("the Grok OAuth callback server stopped unexpectedly")??;
if callback.state != pkce.state {
bail!("the authorization response state did not match — aborting to prevent CSRF");
}
exchange_code_for_tokens(&callback.code, &pkce.verifier).await
}
/// Blocks (on a non-blocking listener with polling) until the browser hits the
/// redirect URI, returning the captured code and state, or an error on timeout.
fn wait_for_callback(listener: &TcpListener, timeout: Duration) -> anyhow::Result<CallbackData> {
let deadline = Instant::now() + timeout;
loop {
if Instant::now() >= deadline {
bail!("timed out waiting for the Grok authorization callback");
}
match listener.accept() {
Ok((stream, _)) => match handle_callback_connection(stream)? {
Some(data) => return Ok(data),
// Unrelated request (e.g. /favicon.ico); keep waiting.
None => continue,
},
Err(ref e) if e.kind() == ErrorKind::WouldBlock => {
std::thread::sleep(POLL_INTERVAL);
}
Err(e) => {
return Err(anyhow::Error::new(e).context("Grok OAuth callback accept failed"))
}
}
}
}
/// Reads a single HTTP request from the callback connection, writes back a
/// minimal HTML response, and extracts the OAuth parameters.
///
/// Returns `Ok(None)` for requests that aren't the OAuth callback (so the
/// caller keeps listening), `Ok(Some(..))` on a successful callback, and `Err`
/// when the provider reported an error or the callback was malformed.
fn handle_callback_connection(mut stream: TcpStream) -> anyhow::Result<Option<CallbackData>> {
// The accepted stream may inherit the listener's non-blocking flag on some
// platforms; force blocking reads with a timeout so we get the full request
// line without spinning.
stream.set_nonblocking(false).ok();
stream.set_read_timeout(Some(Duration::from_secs(10))).ok();
let mut buf = [0u8; 8192];
let n = stream
.read(&mut buf)
.context("failed to read the Grok OAuth callback request")?;
let request = String::from_utf8_lossy(&buf[..n]);
let origin = request_header(&request, "Origin");
// The request line looks like: "GET /callback?code=...&state=... HTTP/1.1".
let mut request_line_parts = request
.lines()
.next()
.unwrap_or_default()
.split_whitespace();
let method = request_line_parts.next().unwrap_or_default();
let path = request_line_parts.next().unwrap_or_default();
if method == "OPTIONS" && path.starts_with("/callback") {
write_response(&mut stream, "204 No Content", "", origin.as_deref());
return Ok(None);
}
let Some(query) = path
.strip_prefix("/callback")
.and_then(|rest| rest.strip_prefix('?'))
else {
write_response(
&mut stream,
"404 Not Found",
"Not found.",
origin.as_deref(),
);
return Ok(None);
};
let mut code = None;
let mut state = None;
let mut error = None;
let mut error_description = None;
let pairs: Vec<(String, String)> = serde_urlencoded::from_str(query).unwrap_or_default();
for (key, value) in pairs {
match key.as_str() {
"code" => code = Some(value),
"state" => state = Some(value),
"error" => error = Some(value),
"error_description" => error_description = Some(value),
_ => {}
}
}
if let Some(error) = error {
write_response(
&mut stream,
"400 Bad Request",
FAILURE_HTML,
origin.as_deref(),
);
let detail = error_description.unwrap_or(error);
bail!("Grok authorization was denied or failed: {detail}");
}
let (Some(code), Some(state)) = (code, state) else {
write_response(
&mut stream,
"400 Bad Request",
FAILURE_HTML,
origin.as_deref(),
);
bail!("the Grok authorization callback was missing the code or state parameter");
};
write_response(&mut stream, "200 OK", SUCCESS_HTML, origin.as_deref());
Ok(Some(CallbackData { code, state }))
}
fn request_header(request: &str, header_name: &str) -> Option<String> {
request.lines().skip(1).find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case(header_name)
.then(|| value.trim().to_owned())
})
}
/// Writes a minimal HTTP/1.1 response and closes the connection.
fn write_response(stream: &mut TcpStream, status: &str, body: &str, origin: Option<&str>) {
let cors_headers = cors_headers(origin);
let response = format!(
"HTTP/1.1 {status}\r\nContent-Type: text/html; charset=utf-8\r\n\
{cors_headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.flush();
let _ = stream.shutdown(Shutdown::Both);
}
fn cors_headers(origin: Option<&str>) -> String {
origin
.filter(|origin| CORS_ALLOWED_ORIGINS.contains(origin))
.map(|origin| {
format!(
"Access-Control-Allow-Origin: {origin}\r\n\
Access-Control-Allow-Methods: GET, OPTIONS\r\n\
Access-Control-Allow-Headers: Content-Type\r\n\
Access-Control-Allow-Private-Network: true\r\n\
Vary: Origin\r\n"
)
})
.unwrap_or_default()
}
/// Exchanges the authorization code for OAuth tokens at xAI's token endpoint.
async fn exchange_code_for_tokens(code: &str, verifier: &str) -> anyhow::Result<TokenResponse> {
let redirect = redirect_uri();
let form: [(&str, &str); 5] = [
("grant_type", "authorization_code"),
("code", code),
("redirect_uri", &redirect),
("client_id", CLIENT_ID),
("code_verifier", verifier),
];
post_token_request(&form).await
}
/// Exchanges a previously obtained refresh token for a fresh set of tokens via
/// the OAuth 2.0 `refresh_token` grant. Used to keep the connected Grok
/// subscription's access token valid without re-running the browser flow.
///
/// xAI may or may not return a new `refresh_token`; callers should fall back to
/// the existing one when [`TokenResponse::refresh_token`] is `None` (rotation is
/// optional in OAuth 2.0).
pub async fn refresh_access_token(refresh_token: &str) -> anyhow::Result<TokenResponse> {
let form: [(&str, &str); 3] = [
("grant_type", "refresh_token"),
("refresh_token", refresh_token),
("client_id", CLIENT_ID),
];
post_token_request(&form).await
}
/// POSTs a form-encoded body to xAI's token endpoint and parses the
/// [`TokenResponse`]. Shared by the initial code exchange and refresh grants.
async fn post_token_request<T: serde::Serialize + ?Sized>(
form: &T,
) -> anyhow::Result<TokenResponse> {
let response = http_client::Client::new()
.post(TOKEN_URL)
.form(form)
.send()
.await
.context("failed to send the Grok token request")?;
let status = response.status();
if !status.is_success() {
let body = response.text().await.unwrap_or_default();
bail!("Grok token request failed ({status}): {body}");
}
response
.json::<TokenResponse>()
.await
.context("failed to parse the Grok token response")
}
const SUCCESS_HTML: &str = "<!doctype html><html><head><meta charset=\"utf-8\">\
<title>Warp — Grok connected</title></head>\
<body style=\"font-family:system-ui,-apple-system,sans-serif;text-align:center;padding:3rem\">\
<h1>Grok connected</h1><p>You can close this window and return to Warp.</p></body></html>";
const FAILURE_HTML: &str = "<!doctype html><html><head><meta charset=\"utf-8\">\
<title>Warp — Grok authorization failed</title></head>\
<body style=\"font-family:system-ui,-apple-system,sans-serif;text-align:center;padding:3rem\">\
<h1>Authorization failed</h1><p>Something went wrong. Return to Warp and try again.</p></body></html>";
#[cfg(test)]
#[path = "oauth_tests.rs"]
mod tests;
@@ -0,0 +1,57 @@
use super::*;
#[test]
fn authorize_url_contains_required_params() {
let pkce = PkceParams::generate();
let url = authorize_url(&pkce);
assert!(url.starts_with("https://auth.x.ai/oauth2/authorize?"));
assert!(url.contains("response_type=code"));
assert!(url.contains(&format!("client_id={CLIENT_ID}")));
assert!(url.contains("code_challenge_method=S256"));
assert!(url.contains("scope=openid"));
assert!(url.contains("plan=generic"));
assert!(url.contains("referrer=warp"));
// The redirect URI must be percent-encoded and match the registered value.
assert!(url.contains("redirect_uri=http%3A%2F%2F127.0.0.1%3A56121%2Fcallback"));
// The CSRF state and PKCE challenge are echoed into the URL verbatim
// (both are URL-safe base64, so no percent-encoding is applied).
assert!(url.contains(&format!("state={}", pkce.state)));
assert!(url.contains(&format!("code_challenge={}", pkce.challenge)));
}
#[test]
fn token_response_parses_minimal_and_full() {
let minimal: TokenResponse =
serde_json::from_str(r#"{"access_token":"abc"}"#).expect("minimal response should parse");
assert_eq!(minimal.access_token, "abc");
assert!(minimal.refresh_token.is_none());
assert!(minimal.expires_in.is_none());
// Unconsumed response fields (token_type, scope) are ignored by serde.
let full: TokenResponse = serde_json::from_str(
r#"{"access_token":"a","refresh_token":"r","token_type":"Bearer","expires_in":3600,"scope":"api:access"}"#,
)
.expect("full response should parse");
assert_eq!(full.access_token, "a");
assert_eq!(full.refresh_token.as_deref(), Some("r"));
assert_eq!(full.expires_in, Some(3600));
}
#[test]
fn manual_code_exchange_captures_attempt_verifier() {
let pkce = PkceParams::generate();
let exchange = ManualCodeExchange {
verifier: pkce.verifier.clone(),
};
assert_eq!(exchange.verifier, pkce.verifier);
}
#[test]
fn manual_code_exchange_rejects_blank_code() {
let exchange = ManualCodeExchange {
verifier: "verifier".to_string(),
};
let result = galaxyui_core::r#async::block_on(exchange.exchange(" "));
assert!(result.is_err());
}
+5 -4
View File
@@ -7,14 +7,15 @@ cfg_if::cfg_if! {
}
}
use crate::index::{Entry, FileId};
use ignore::gitignore::Gitignore;
use std::collections::{HashMap, VecDeque};
use std::path::PathBuf;
use ignore::gitignore::Gitignore;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use crate::index::{Entry, FileId};
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct FileSymbols {
pub path: String,
@@ -43,7 +44,7 @@ impl Outline {
let mut queue = VecDeque::from([&self.root]);
let mut repo_map = Vec::new();
// Iteratively print the files while perserving their traversal order.
// Iteratively print the files while preserving their traversal order.
while let Some(entry) = queue.pop_front() {
match entry {
Entry::Directory(directory) => {
@@ -91,7 +92,7 @@ impl Outline {
let mut queue = VecDeque::from([&self.root]);
let mut file_to_symbols = HashMap::new();
// Iteratively print the files while perserving their traversal order.
// Iteratively print the files while preserving their traversal order.
while let Some(entry) = queue.pop_front() {
match entry {
Entry::Directory(directory) => {
+19 -207
View File
@@ -1,21 +1,21 @@
use futures::channel::oneshot;
use ignore::gitignore::Gitignore;
use rayon::prelude::*;
use repo_metadata::entry::is_file_parsable;
use repo_metadata::RepositoryUpdate;
use std::collections::HashMap;
use std::{fs, path::Path};
use std::fs;
use std::path::Path;
use anyhow::anyhow;
use arborium::tree_sitter::{Parser, Query, QueryCursor, Tree};
use futures::channel::oneshot;
use ignore::gitignore::Gitignore;
use itertools::Itertools;
use rayon::prelude::*;
use repo_metadata::entry::{is_file_parsable, BudgetExceededBehavior, IgnoredPathStrategy};
use repo_metadata::RepositoryUpdate;
use streaming_iterator::StreamingIterator;
use syntax_tree::TextSlice;
use warp_util::standardized_path::StandardizedPath;
use crate::index::file_outline::{FileOutline, Outline, Symbol};
use crate::index::THREADPOOL;
use crate::index::{Entry, FileId, FileMetadata};
use repo_metadata::entry::IgnoredPathStrategy;
use crate::index::{Entry, FileId, FileMetadata, THREADPOOL};
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
@@ -55,6 +55,7 @@ pub async fn build_outline(
MAX_DEPTH,
0,
&IgnoredPathStrategy::Exclude, // override_ignore_for_files
BudgetExceededBehavior::StopAndLazyLoad,
)?;
let (sender, receiver) = oneshot::channel();
@@ -231,7 +232,8 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
if !is_file_parsable(path)? {
return Err(anyhow!("File exceeds max file size limit for parsing"));
}
let Some(language) = languages::language_by_filename(path) else {
let standardized_path = StandardizedPath::try_from_local(path)?;
let Some(language) = languages::language_by_filename(&standardized_path) else {
return Err(anyhow!("Language unsupported for file {:?}", path));
};
let content = fs::read_to_string(path)?;
@@ -267,7 +269,11 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
// of the allocator).
//
// See: https://github.com/tree-sitter/tree-sitter/issues/3129
#[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
#[cfg(all(
any(target_os = "linux", target_os = "freebsd"),
target_env = "gnu",
not(feature = "jemalloc")
))]
unsafe {
nix::libc::malloc_trim(0);
}
@@ -333,199 +339,5 @@ fn get_symbols<'a>(
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_file(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
let file_path = dir.path().join(filename);
let mut file = File::create(&file_path).unwrap();
file.write_all(content.as_bytes()).unwrap();
file_path
}
#[test]
fn test_parse_comments() {
let temp_dir = TempDir::new().unwrap();
let content = r#"
/// This is a struct for NewFunc
struct NewFunc {
a: str,
}
// Hello
// World
fn first_function() {
println!("First");
}
impl NewFunc {
fn second_function() {
println!("Second");
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols[0].name, "NewFunc");
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
assert_eq!(
symbols[0].comment,
Some(vec!["/// This is a struct for NewFunc".to_owned()])
);
assert_eq!(symbols[0].line_number, 3); // struct NewFunc is on line 3
assert_eq!(symbols[1].name, "first_function");
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[1].line_number, 9); // first_function is on line 9
assert_eq!(symbols[2].name, "second_function");
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[2].line_number, 14); // second_function is on line 14
}
#[test]
fn test_parse_multiple_languages() {
let temp_dir = TempDir::new().unwrap();
let content = r#"
struct NewFunc {
a: str,
}
fn first_function() {
println!("First");
}
impl NewFunc {
fn second_function() {
println!("Second");
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 3);
assert_eq!(symbols[0].name, "NewFunc");
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
assert_eq!(symbols[1].name, "first_function");
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[2].name, "second_function");
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
// Test parsing Python code with multiple symbol definitions
// This verifies parsing of:
// - Regular function definitions (def keyword)
// - Class definitions (class keyword)
// - Method definitions within a class (def keyword)
let python_content = r#"
def first_function():
print("First")
class TestClass:
def __init__(self):
pass
def class_method(self):
print("Method")
def second_function():
print("Second")
"#;
let file_path = create_test_file(&temp_dir, "multiple.py", python_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 5);
assert_eq!(symbols[0].name, "first_function");
assert_eq!(symbols[0].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[1].name, "TestClass");
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
assert_eq!(symbols[2].name, "__init__");
assert_eq!(symbols[2].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[3].name, "class_method");
assert_eq!(symbols[3].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[4].name, "second_function");
assert_eq!(symbols[4].type_prefix, Some("def".to_owned()));
// Test parsing JavaScript code with multiple symbol definitions
// This verifies parsing of:
// - Function declarations
// - Class declarations
// - Method definitions
// - Arrow functions assigned to variables
let js_content = r#"
function regularFunction() {
console.log('Regular function');
}
class TestClass {
constructor() {
this.value = 42;
}
classMethod() {
return this.value;
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.js", js_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 4);
assert_eq!(symbols[0].name, "regularFunction");
assert_eq!(symbols[0].type_prefix, Some("function".to_owned()));
assert_eq!(symbols[1].name, "TestClass");
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
assert_eq!(symbols[2].name, "constructor");
assert_eq!(symbols[2].type_prefix, None);
assert_eq!(symbols[3].name, "classMethod");
assert_eq!(symbols[3].type_prefix, None);
// Test parsing Go code with multiple symbol definitions
// This verifies parsing of:
// - Function definitions (func keyword)
// - Type definitions (struct, interface)
// - Method definitions (func with receiver)
let go_content = r#"
package main
func mainFunction() {
fmt.Println("Main function")
}
type TestStruct struct {
field string
}
func (t *TestStruct) structMethod() string {
return t.field
}
type TestInterface interface {
InterfaceMethod() string
}
func helperFunction() {
fmt.Println("Helper function")
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.go", go_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 5);
assert_eq!(symbols[0].name, "mainFunction");
assert_eq!(symbols[0].type_prefix, Some("func".to_owned()));
assert_eq!(symbols[1].name, "TestStruct");
assert_eq!(symbols[1].type_prefix, Some("type".to_owned()));
assert_eq!(symbols[2].name, "structMethod");
assert_eq!(symbols[2].type_prefix, Some("func".to_owned()));
assert_eq!(symbols[3].name, "TestInterface");
assert_eq!(symbols[3].type_prefix, Some("type".to_owned()));
assert_eq!(symbols[4].name, "helperFunction");
assert_eq!(symbols[4].type_prefix, Some("func".to_owned()));
}
}
#[path = "native_tests.rs"]
mod tests;
@@ -0,0 +1,196 @@
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use tempfile::TempDir;
use super::*;
fn create_test_file(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
let file_path = dir.path().join(filename);
let mut file = File::create(&file_path).unwrap();
file.write_all(content.as_bytes()).unwrap();
file_path
}
#[test]
fn test_parse_comments() {
let temp_dir = TempDir::new().unwrap();
let content = r#"
/// This is a struct for NewFunc
struct NewFunc {
a: str,
}
// Hello
// World
fn first_function() {
println!("First");
}
impl NewFunc {
fn second_function() {
println!("Second");
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols[0].name, "NewFunc");
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
assert_eq!(
symbols[0].comment,
Some(vec!["/// This is a struct for NewFunc".to_owned()])
);
assert_eq!(symbols[0].line_number, 3); // struct NewFunc is on line 3
assert_eq!(symbols[1].name, "first_function");
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[1].line_number, 9); // first_function is on line 9
assert_eq!(symbols[2].name, "second_function");
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[2].line_number, 14); // second_function is on line 14
}
#[test]
fn test_parse_multiple_languages() {
let temp_dir = TempDir::new().unwrap();
let content = r#"
struct NewFunc {
a: str,
}
fn first_function() {
println!("First");
}
impl NewFunc {
fn second_function() {
println!("Second");
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 3);
assert_eq!(symbols[0].name, "NewFunc");
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
assert_eq!(symbols[1].name, "first_function");
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
assert_eq!(symbols[2].name, "second_function");
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
// Test parsing Python code with multiple symbol definitions
// This verifies parsing of:
// - Regular function definitions (def keyword)
// - Class definitions (class keyword)
// - Method definitions within a class (def keyword)
let python_content = r#"
def first_function():
print("First")
class TestClass:
def __init__(self):
pass
def class_method(self):
print("Method")
def second_function():
print("Second")
"#;
let file_path = create_test_file(&temp_dir, "multiple.py", python_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 5);
assert_eq!(symbols[0].name, "first_function");
assert_eq!(symbols[0].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[1].name, "TestClass");
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
assert_eq!(symbols[2].name, "__init__");
assert_eq!(symbols[2].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[3].name, "class_method");
assert_eq!(symbols[3].type_prefix, Some("def".to_owned()));
assert_eq!(symbols[4].name, "second_function");
assert_eq!(symbols[4].type_prefix, Some("def".to_owned()));
// Test parsing JavaScript code with multiple symbol definitions
// This verifies parsing of:
// - Function declarations
// - Class declarations
// - Method definitions
// - Arrow functions assigned to variables
let js_content = r#"
function regularFunction() {
console.log('Regular function');
}
class TestClass {
constructor() {
this.value = 42;
}
classMethod() {
return this.value;
}
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.js", js_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 4);
assert_eq!(symbols[0].name, "regularFunction");
assert_eq!(symbols[0].type_prefix, Some("function".to_owned()));
assert_eq!(symbols[1].name, "TestClass");
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
assert_eq!(symbols[2].name, "constructor");
assert_eq!(symbols[2].type_prefix, None);
assert_eq!(symbols[3].name, "classMethod");
assert_eq!(symbols[3].type_prefix, None);
// Test parsing Go code with multiple symbol definitions
// This verifies parsing of:
// - Function definitions (func keyword)
// - Type definitions (struct, interface)
// - Method definitions (func with receiver)
let go_content = r#"
package main
func mainFunction() {
fmt.Println("Main function")
}
type TestStruct struct {
field string
}
func (t *TestStruct) structMethod() string {
return t.field
}
type TestInterface interface {
InterfaceMethod() string
}
func helperFunction() {
fmt.Println("Helper function")
}
"#;
let file_path = create_test_file(&temp_dir, "multiple.go", go_content);
let outline = parse_file_outline(&file_path).unwrap();
let symbols = outline.symbols.unwrap();
assert_eq!(symbols.len(), 5);
assert_eq!(symbols[0].name, "mainFunction");
assert_eq!(symbols[0].type_prefix, Some("func".to_owned()));
assert_eq!(symbols[1].name, "TestStruct");
assert_eq!(symbols[1].type_prefix, Some("type".to_owned()));
assert_eq!(symbols[2].name, "structMethod");
assert_eq!(symbols[2].type_prefix, Some("func".to_owned()));
assert_eq!(symbols[3].name, "TestInterface");
assert_eq!(symbols[3].type_prefix, Some("type".to_owned()));
assert_eq!(symbols[4].name, "helperFunction");
assert_eq!(symbols[4].type_prefix, Some("func".to_owned()));
}
@@ -46,5 +46,5 @@ impl ChangedFiles {
}
#[cfg(test)]
#[path = "changed_files_test.rs"]
#[path = "changed_files_tests.rs"]
mod tests;
@@ -1,6 +1,7 @@
use super::*;
use std::path::PathBuf;
use super::*;
// Helper function to create a PathBuf from a string
fn pb(path: &str) -> PathBuf {
PathBuf::from(path)
@@ -1,6 +1,8 @@
use std::path::Path;
use string_offset::ByteOffset;
#[cfg(not(target_family = "wasm"))]
use warp_util::standardized_path::StandardizedPath;
mod naive;
#[cfg(not(target_family = "wasm"))]
@@ -99,7 +101,8 @@ pub fn chunk_code<'a>(code: &'a str, path: &'a Path) -> Vec<Fragment<'a>> {
/// could not be chunked for any reason.
#[cfg(not(target_family = "wasm"))]
fn try_chunk_code_semantically<'a>(code: &'a str, path: &'a Path) -> Option<Vec<Fragment<'a>>> {
let language = languages::language_by_filename(path)?;
let standardized_path = StandardizedPath::try_from_local(path).ok()?;
let language = languages::language_by_filename(&standardized_path)?;
semantic::chunk_code(code, path, MAX_BYTES_PER_CHUNK, &language.grammar).ok()
}
@@ -1,7 +1,9 @@
use crate::index::full_source_code_embedding::chunker::{coalesce_fragments, Fragment};
use std::path::Path;
use itertools::Itertools;
use line_span::{LineSpan, LineSpans};
use std::path::Path;
use crate::index::full_source_code_embedding::chunker::{coalesce_fragments, Fragment};
/// Chunks the given file into [`Fragment`]s. Each chunk is at most `num_lines_per_chunk` lines long, and contains at most `max_bytes_per_chunk` bytes.
pub(super) fn chunk_code<'a>(
@@ -1,6 +1,7 @@
use super::*;
use std::path::Path;
use super::*;
#[test]
fn test_chunker() {
let code = "This is some text content\nthat should be chunked\nusing the naive chunker\nbecause the language isn't recognized.";
@@ -48,7 +48,11 @@ pub(super) fn chunk_code<'a>(
// of the allocator).
//
// See: https://github.com/tree-sitter/tree-sitter/issues/3129
#[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
#[cfg(all(
any(target_os = "linux", target_os = "freebsd"),
target_env = "gnu",
not(feature = "jemalloc")
))]
unsafe {
nix::libc::malloc_trim(0);
}
@@ -1,6 +1,7 @@
use std::path::Path;
use languages::language_by_filename;
use warp_util::standardized_path::StandardizedPath;
use super::*;
@@ -33,12 +34,14 @@ fn main() {
"#;
let max_chunk_size = 128;
let language_path =
StandardizedPath::try_new("/test.rs").expect("test path should be absolute");
let chunks = chunk_code(
source_code,
Path::new("test.rs"),
max_chunk_size,
&language_by_filename(Path::new("test.rs"))
&language_by_filename(&language_path)
.expect("Rust language must exist")
.grammar,
)
@@ -1,3 +1,9 @@
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use anyhow::anyhow;
use async_channel;
use chrono::{DateTime, Utc};
@@ -5,32 +11,31 @@ use futures::stream::AbortHandle;
use galaxy_core::safe_error;
use galaxyui::{Entity, ModelContext, ModelHandle};
use ignore::gitignore::Gitignore;
use instant::Instant;
#[cfg(feature = "local_fs")]
use repo_metadata::entry::IgnoredPathStrategy;
use repo_metadata::entry::{BudgetExceededBehavior, IgnoredPathStrategy};
use repo_metadata::Repository;
use std::{path::Path, sync::Arc};
use galaxyui_core::{Entity, ModelContext, ModelHandle};
use super::fragment_metadata::{
FragmentMetadata, LeafToFragmentMetadata, LeafToFragmentMetadataUpdates,
};
use super::manager::{
CodebaseIndexFinishedStatus, CodebaseIndexStatus, FragmentMetadataLookupError,
RetrieveFileError,
};
use super::merkle_tree::{MerkleTree, SerializedCodebaseIndex};
#[cfg(feature = "local_fs")]
use super::search_shaping::build_fragments_from_file_contents;
use super::search_shaping::{fragments_to_context_locations, ReadFragmentResult};
use super::store_client::StoreClient;
use super::sync_client::{FlushFragmentResult, SyncOperationError};
use super::{
fragment_metadata::{FragmentMetadata, LeafToFragmentMetadata, LeafToFragmentMetadataUpdates},
manager::{CodebaseIndexFinishedStatus, CodebaseIndexStatus, RetrieveFileError},
merkle_tree::{MerkleTree, SerializedCodebaseIndex},
store_client::StoreClient,
sync_client::{FlushFragmentResult, SyncOperationError},
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
};
use crate::{
index::locations::{CodeContextLocation, FileFragmentLocation},
telemetry::{AITelemetryEvent, CodebaseContextSyncType},
workspace::{WorkspaceMetadata, WorkspaceMetadataEvent},
};
use instant::Instant;
use std::{
collections::{HashMap, HashSet},
ops::Range,
path::PathBuf,
sync::atomic::{AtomicUsize, Ordering},
time::Duration,
};
use crate::index::locations::CodeContextLocation;
use crate::telemetry::{AITelemetryEvent, CodebaseContextSyncType};
use crate::workspace::{WorkspaceMetadata, WorkspaceMetadataEvent};
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
@@ -44,12 +49,11 @@ cfg_if::cfg_if! {
Entry,
matches_gitignores,
full_source_code_embedding::sync_client::CodebaseIndexSyncOperation,
full_source_code_embedding::FragmentLocation
};
use galaxy_core::send_telemetry_from_ctx;
use galaxy_core::interval_timer::IntervalTimer;
use galaxyui::r#async::Timer;
use galaxyui::SingletonEntity;
use galaxyui_core::r#async::Timer;
use galaxyui_core::SingletonEntity;
use galaxy_core::sync_queue::SyncQueue;
use sha2::Digest;
}
@@ -316,7 +320,9 @@ pub enum CodebaseIndexEvent {
retrieval_id: RetrievalID,
error: Error,
},
SyncStateUpdated,
SyncStateUpdated {
root_path: PathBuf,
},
IndexMetadataUpdated {
root_path: PathBuf,
event: WorkspaceMetadataEvent,
@@ -498,6 +504,13 @@ impl CodebaseIndex {
store_client: Arc<dyn StoreClient>,
ctx: &mut ModelContext<Self>,
) {
if self
.pending_file_changes
.as_ref()
.is_none_or(|changed_files| changed_files.is_empty())
{
return;
}
let last_server_synced_root_node = self.last_server_synced_root_node();
let old_state = self.update_tree_sync_state(
TreeSourceSyncState::Syncing {
@@ -876,7 +889,9 @@ impl CodebaseIndex {
ctx: &mut ModelContext<Self>,
) -> TreeSourceSyncState {
let old_state = std::mem::replace(&mut self.tree_sync_state, new_state);
ctx.emit(CodebaseIndexEvent::SyncStateUpdated);
ctx.emit(CodebaseIndexEvent::SyncStateUpdated {
root_path: self.repo_path.clone(),
});
old_state
}
@@ -885,7 +900,9 @@ impl CodebaseIndex {
if let TreeSourceSyncState::Syncing { sync_progress, .. } = &mut self.tree_sync_state {
*sync_progress = Some(progress);
ctx.emit(CodebaseIndexEvent::SyncStateUpdated);
ctx.emit(CodebaseIndexEvent::SyncStateUpdated {
root_path: self.repo_path.clone(),
});
}
}
@@ -920,6 +937,9 @@ impl CodebaseIndex {
// First traverse the repo path to retrieve all files we want to parse.
let mut files = Vec::new();
let mut remaining_file_quotas = max_num_files_limit;
// Codebase embedding must not operate on a partial tree: the file limit
// is an intentional cost cap, so exceeding it fails the build rather
// than silently indexing a breadth-first subset of the repository.
let entry = Entry::build_tree(
&repo_path,
&mut files,
@@ -928,6 +948,7 @@ impl CodebaseIndex {
MAX_DEPTH,
0,
&IgnoredPathStrategy::Exclude, // override_ignore_for_files
BudgetExceededBehavior::FailFast,
)?;
Ok(BuildFileTreeResult {
@@ -1318,6 +1339,30 @@ impl CodebaseIndex {
self.leaf_node_to_fragment_metadatas.get(leaf_hash.as_ref())
}
pub(super) fn fragment_metadatas_from_hashes(
&self,
root_hash: &NodeHash,
content_hashes: &[ContentHash],
) -> Result<HashMap<ContentHash, Vec<FragmentMetadata>>, FragmentMetadataLookupError> {
let current_root_hash = self
.last_server_synced_root_node()
.ok_or(FragmentMetadataLookupError::IndexNotSynced)?;
if &current_root_hash != root_hash {
return Err(FragmentMetadataLookupError::RootHashMismatch {
requested: root_hash.clone(),
current: current_root_hash,
});
}
Ok(content_hashes
.iter()
.filter_map(|hash| {
self.fragment_metadatas_from_hash(hash)
.map(|metadata| (hash.clone(), metadata.clone()))
})
.collect())
}
fn repo_metadata(&self) -> RepoMetadata {
RepoMetadata {
path: Some(self.repo_path.to_string_lossy().to_string()),
@@ -1342,12 +1387,20 @@ impl CodebaseIndex {
}
pub(super) fn codebase_index_status(&self) -> CodebaseIndexStatus {
let has_synced_version = self.last_server_synced_root_node().is_some();
let root_hash = self.last_server_synced_root_node();
let has_synced_version = root_hash.is_some();
#[cfg(feature = "local_fs")]
let has_pending_file_changes = self
.pending_file_changes
.as_ref()
.is_some_and(|changes| !changes.is_empty());
#[cfg(not(feature = "local_fs"))]
let has_pending_file_changes = false;
match &self.tree_sync_state {
TreeSourceSyncState::Synced {
server_sync_result, ..
} => CodebaseIndexStatus {
has_pending: false,
has_pending: has_pending_file_changes,
has_synced_version,
last_sync_successful: Some(match server_sync_result {
ServerSyncResult::Success => CodebaseIndexFinishedStatus::Completed,
@@ -1356,18 +1409,21 @@ impl CodebaseIndex {
}
}),
sync_progress: None,
root_hash: root_hash.clone(),
},
TreeSourceSyncState::InitializeTreeFailure(e) => CodebaseIndexStatus {
has_pending: false,
has_synced_version,
last_sync_successful: Some(CodebaseIndexFinishedStatus::Failed(e.into())),
sync_progress: None,
root_hash: root_hash.clone(),
},
TreeSourceSyncState::Syncing { sync_progress, .. } => CodebaseIndexStatus {
has_pending: true,
has_synced_version,
last_sync_successful: None,
sync_progress: *sync_progress,
root_hash: root_hash.clone(),
},
}
}
@@ -1622,82 +1678,21 @@ impl CodebaseIndex {
}
}
// Convert fragments into CodeContextLocations. This function groups and dedupes fragments in the same file.
// It also allows the caller to define a context line number surrounding the relevant fragment.
fn process_fragments(
&self,
fragments: Vec<Fragment>,
context_lines: usize,
) -> HashSet<CodeContextLocation> {
// Map to collect fragments by file path
let mut fragments_by_path: HashMap<&PathBuf, Vec<Range<usize>>> = HashMap::new();
let mut whole_files = HashSet::new();
// First pass - collect all fragments and their line ranges by file path
for fragment in &fragments {
if let Some(metadata) = self
.fragment_metadatas_from_hash(&fragment.content_hash)
.and_then(|metadatas| {
metadatas.iter().find(|m| {
m.absolute_path == fragment.location.absolute_path
&& m.location.byte_range == fragment.location.byte_range
})
})
{
// Add line range with context to the appropriate file's collection
let path = &fragment.location.absolute_path;
let start = metadata.location.start_line.saturating_sub(context_lines);
let end = metadata.location.end_line + 1 + context_lines; // Make the range inclusive on both ends
fragments_by_path.entry(path).or_default().push(start..end);
} else {
// Fallback to whole file if metadata not found
whole_files.insert(fragment.location.absolute_path.clone());
}
}
// Second pass - process each file's fragments
let mut result = HashSet::new();
// Process each file's fragments
for (path, mut line_ranges) in fragments_by_path {
if line_ranges.is_empty() {
continue;
}
// We can skip the fragments if the entire file is already included in the context.
if whole_files.contains(path) {
continue;
}
// Sort ranges by start position
line_ranges.sort_by_key(|range| range.start);
// Merge overlapping or adjacent ranges
let mut merged_ranges: Vec<Range<usize>> = Vec::new();
for range in line_ranges {
if let Some(last) = merged_ranges.last_mut() {
// If current range overlaps or is adjacent to the last one, merge them
if range.start <= last.end {
last.end = last.end.max(range.end);
} else {
merged_ranges.push(range);
}
} else {
merged_ranges.push(range);
}
}
// Add file fragment location with all merged ranges
result.insert(CodeContextLocation::Fragment(FileFragmentLocation {
path: path.clone(),
line_ranges: merged_ranges,
}));
}
// Add whole files to the result set
result.extend(whole_files.into_iter().map(CodeContextLocation::WholeFile));
result
// Keep local and remote search aligned by using the same fragment-to-context expansion
// helper for range merging, deduping, and context-line handling.
fragments_to_context_locations(
fragments,
|content_hash| {
self.fragment_metadatas_from_hash(content_hash)
.map(Vec::as_slice)
},
context_lines,
)
}
/// A new index built from a snapshot. This constructor builds the index and starts
@@ -2100,13 +2095,14 @@ impl CodebaseIndex {
match entry.and_then(|entry| dunce::canonicalize(entry.path())) {
Ok(child_path) => {
// Ignore paths that are excluded by .gitignore, end with .git, or are symlinks.
if matches_gitignores(
&child_path,
is_dir,
&*gitignores,
false, /* check_ancestors */
) || child_path.ends_with(".git")
if child_path.ends_with(".git")
|| child_path.is_symlink()
|| matches_gitignores(
&child_path,
child_path.is_dir(),
&*gitignores,
false, /* check_ancestors */
)
{
continue;
}
@@ -2244,13 +2240,14 @@ impl CodebaseIndex {
match entry.and_then(|entry| dunce::canonicalize(entry.path())) {
Ok(child_path) => {
// Ignore paths that are excluded by .gitignore, end with .git, or are symlinks.
if matches_gitignores(
&child_path,
is_dir,
&*gitignores,
false, /* check_ancestors */
) || child_path.ends_with(".git")
if child_path.ends_with(".git")
|| child_path.is_symlink()
|| matches_gitignores(
&child_path,
child_path.is_dir(),
&*gitignores,
false, /* check_ancestors */
)
{
continue;
}
@@ -2320,98 +2317,22 @@ impl CodebaseIndex {
}
}
#[derive(Default)]
pub struct ReadFragmentResult {
pub successfully_read: Vec<Fragment>,
pub fail_to_read: Vec<ContentHash>,
pub fail_to_read_path: Vec<PathBuf>,
}
#[cfg(feature = "local_fs")]
pub(super) async fn build_fragments_from_metadata(
metadatas: impl IntoIterator<Item = (ContentHash, FragmentMetadata)>,
) -> ReadFragmentResult {
let mut fragments = Vec::new();
let mut fail_to_read = Vec::new();
let mut fail_to_read_path = Vec::new();
// Group fragments by file path
let mut fragments_by_path: HashMap<_, Vec<_>> = HashMap::new();
for (content_hash, metadata) in metadatas {
fragments_by_path
.entry(metadata.absolute_path)
.or_default()
.push((content_hash, metadata.location.byte_range));
}
// Process each file and its fragments
for (file_path, file_fragments) in fragments_by_path {
let mut has_failed_to_read_fragments = false;
// Read the file content once
if let Ok(file_content) = async_fs::read_to_string(&file_path).await {
// Process all fragments for this file
for (content_hash, fragment_ranges) in file_fragments {
let start_idx = fragment_ranges.start.as_usize();
let end_idx = fragment_ranges.end.as_usize();
if start_idx <= end_idx
&& end_idx <= file_content.len()
&& file_content.is_char_boundary(start_idx)
&& file_content.is_char_boundary(end_idx)
{
let content = file_content[start_idx..end_idx].to_string();
if content.is_empty() {
log::trace!(
"Fragment for {:?} with range {:?} is empty",
file_path.display(),
fragment_ranges
);
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
} else if ContentHash::from_content(&content) != content_hash {
log::trace!(
"Fragment for {:?} with range {:?} does not match its content hash",
file_path.display(),
fragment_ranges
);
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
} else {
fragments.push(Fragment {
content,
content_hash,
location: FragmentLocation {
absolute_path: file_path.clone(),
byte_range: fragment_ranges,
},
});
}
} else {
log::trace!("Invalid byte range {fragment_ranges:?} for file: {file_path:?}");
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
}
}
} else {
log::trace!("Failed to read file: {file_path:?}");
fail_to_read.extend(
file_fragments
.into_iter()
.map(|(content_hash, _)| content_hash),
);
has_failed_to_read_fragments = true;
}
if has_failed_to_read_fragments {
fail_to_read_path.push(file_path);
let metadatas = metadatas.into_iter().collect::<Vec<_>>();
let mut file_contents = HashMap::new();
for path in metadatas
.iter()
.map(|(_, metadata)| metadata.absolute_path.clone())
.collect::<HashSet<_>>()
{
if let Ok(file_content) = async_fs::read_to_string(&path).await {
file_contents.insert(path, file_content);
}
}
ReadFragmentResult {
successfully_read: fragments,
fail_to_read,
fail_to_read_path,
}
build_fragments_from_file_contents(metadatas, &file_contents)
}
#[cfg(not(feature = "local_fs"))]
@@ -1,34 +1,32 @@
#![allow(clippy::single_range_in_vec_init)]
use chrono::Utc;
use string_offset::ByteOffset;
use virtual_fs::{Stub, VirtualFS};
use crate::index::full_source_code_embedding::changed_files::ChangedFiles;
use crate::index::full_source_code_embedding::codebase_index::MAX_DEPTH;
use crate::index::full_source_code_embedding::fragment_metadata::{
FragmentLocation, LeafToFragmentMetadata,
};
use crate::index::full_source_code_embedding::merkle_tree::MerkleHash;
use crate::index::full_source_code_embedding::merkle_tree::MerkleTree;
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
use crate::index::full_source_code_embedding::{
ContentHash, EmbeddingConfig, Fragment, FragmentMetadata,
};
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
use futures::executor::block_on;
use galaxy_util::standardized_path::StandardizedPath;
use galaxyui::{App, SingletonEntity};
use repo_metadata::DirectoryWatcher;
use std::collections::HashMap;
use std::ops::Range;
use std::path::PathBuf;
use std::sync::Arc;
use chrono::Utc;
use futures::executor::block_on;
use repo_metadata::DirectoryWatcher;
use string_offset::ByteOffset;
use virtual_fs::{Stub, VirtualFS};
use warp_util::standardized_path::StandardizedPath;
use galaxyui_core::{App, SingletonEntity};
use super::{
CodebaseIndex, CodebaseIndexTimeStampMetadata, TreeSourceSyncState,
CodebaseIndex, CodebaseIndexTimeStampMetadata, ServerSyncResult, TreeSourceSyncState,
DEFAULT_INCREMENAL_SYNC_FLUSH_INTERVAL,
};
use crate::index::full_source_code_embedding::changed_files::ChangedFiles;
use crate::index::full_source_code_embedding::codebase_index::MAX_DEPTH;
use crate::index::full_source_code_embedding::fragment_metadata::{
FragmentLocation, LeafToFragmentMetadata,
};
use crate::index::full_source_code_embedding::merkle_tree::{MerkleHash, MerkleTree};
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
use crate::index::full_source_code_embedding::{
ContentHash, EmbeddingConfig, Fragment, FragmentMetadata,
};
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
impl CodebaseIndex {
fn new_for_test(
@@ -104,6 +102,86 @@ fn create_test_metadata(
}
}
#[test]
fn synced_index_with_queued_file_changes_reports_pending_status() {
VirtualFS::test(
"synced_index_with_queued_file_changes_reports_pending_status",
|dirs, mut sandbox| {
App::test((), |mut app| async move {
app.add_singleton_model(DirectoryWatcher::new);
let repo_name = "warp-virtual";
sandbox.mkdir(repo_name);
sandbox.with_files(vec![Stub::FileWithContent(
format!("{repo_name}/existing_file").as_str(),
"existing content",
)]);
let repo_path = dunce::canonicalize(dirs.tests().join(repo_name)).unwrap();
let build_file_tree_result =
block_on(CodebaseIndex::build_file_tree(repo_path.clone(), None)).unwrap();
let (tree, _) =
block_on(MerkleTree::try_new(build_file_tree_result.file_tree)).unwrap();
let mut index = CodebaseIndex::new_for_test(Default::default(), &mut app);
index.tree_sync_state = TreeSourceSyncState::Synced {
tree,
server_sync_result: ServerSyncResult::Success,
};
let mut changed_files = ChangedFiles::default();
changed_files.upsertions.insert(repo_path.join("new_file"));
index.pending_file_changes = Some(changed_files);
let status = index.codebase_index_status();
assert!(status.has_pending());
assert!(status.has_synced_version());
assert_eq!(status.last_sync_successful(), Some(true));
});
},
);
}
#[test]
fn synced_index_without_pending_file_changes_stays_ready_after_flush() {
VirtualFS::test(
"synced_index_without_pending_file_changes_stays_ready_after_flush",
|dirs, mut sandbox| {
App::test((), |mut app| async move {
app.add_singleton_model(DirectoryWatcher::new);
let repo_name = "warp-virtual";
sandbox.mkdir(repo_name);
sandbox.with_files(vec![Stub::FileWithContent(
format!("{repo_name}/existing_file").as_str(),
"existing content",
)]);
let repo_path = dunce::canonicalize(dirs.tests().join(repo_name)).unwrap();
let build_file_tree_result =
block_on(CodebaseIndex::build_file_tree(repo_path, None)).unwrap();
let (tree, _) =
block_on(MerkleTree::try_new(build_file_tree_result.file_tree)).unwrap();
let mut test_index = CodebaseIndex::new_for_test(Default::default(), &mut app);
test_index.tree_sync_state = TreeSourceSyncState::Synced {
tree,
server_sync_result: ServerSyncResult::Success,
};
let index = app.add_model(|_| test_index);
index.update(&mut app, |index, ctx| {
index.flush_pending_file_changes(ctx);
let status = index.codebase_index_status();
assert!(!status.has_pending());
assert!(status.has_synced_version());
assert_eq!(status.last_sync_successful(), Some(true));
});
});
},
);
}
#[test]
fn test_empty_fragments() {
App::test((), |mut app| async move {
@@ -1,4 +1,6 @@
use std::{collections::HashMap, ops::Range, path::PathBuf};
use std::collections::HashMap;
use std::ops::Range;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use string_offset::ByteOffset;
@@ -1,9 +1,7 @@
use std::{
collections::{HashMap, HashSet},
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use itertools::Itertools;
use repo_metadata::{BuildTreeError, DirectoryWatcher, Repository};
@@ -13,12 +11,12 @@ cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
use chrono::Utc;
use super::changed_files::ChangedFiles;
use crate::index::path_passes_filters;
use crate::index::{is_git_internal_path, matches_gitignores};
use ignore::gitignore::Gitignore;
use notify_debouncer_full::notify::{RecursiveMode, WatchFilter};
use galaxy_core::features::FeatureFlag;
use watcher::{BulkFilesystemWatcher, BulkFilesystemWatcherEvent};
use galaxyui::r#async::Timer;
use galaxyui_core::r#async::Timer;
use galaxy_core::{send_telemetry_from_ctx, report_if_error};
use crate::telemetry::AITelemetryEvent;
use instant::Instant;
@@ -27,21 +25,16 @@ cfg_if::cfg_if! {
}
}
use galaxy_core::safe_anyhow;
use galaxyui::{AppContext, Entity, ModelContext, ModelHandle, SingletonEntity};
use galaxyui_core::{AppContext, Entity, ModelContext, ModelHandle, SingletonEntity};
use super::{
codebase_index::{CodebaseIndexEvent, RetrievalID, SyncProgress},
fragment_metadata::FragmentMetadata,
priority_queue::{BuildQueue, Priority},
snapshot::*,
store_client::StoreClient,
CodebaseIndex, EmbeddingConfig, Error as CodebaseIndexError, NodeHash,
};
use crate::{
index::locations::CodeContextLocation,
workspace::{WorkspaceMetadata, WorkspaceMetadataEvent},
};
use super::codebase_index::{CodebaseIndexEvent, RetrievalID, SyncProgress};
use super::fragment_metadata::FragmentMetadata;
use super::priority_queue::{BuildQueue, Priority};
use super::snapshot::*;
use super::store_client::StoreClient;
use super::{CodebaseIndex, ContentHash, EmbeddingConfig, Error as CodebaseIndexError, NodeHash};
use crate::index::locations::CodeContextLocation;
use crate::workspace::{WorkspaceMetadata, WorkspaceMetadataEvent};
/// The interval for debouncing filesystem events.
const REPO_WATCHER_DEBOUNCE_DURATION: Duration = Duration::from_secs(10);
@@ -69,6 +62,19 @@ pub enum RetrieveFileError {
IndexNotFound,
}
#[derive(Error, Debug)]
pub enum FragmentMetadataLookupError {
#[error("Codebase index not found")]
IndexNotFound,
#[error("Codebase index has no synced root hash")]
IndexNotSynced,
#[error("Codebase index root hash mismatch: requested {requested}, current {current}")]
RootHashMismatch {
requested: NodeHash,
current: NodeHash,
},
}
pub enum CodebaseIndexManagerEvent {
RetrievalRequestCompleted {
retrieval_id: RetrievalID,
@@ -79,7 +85,9 @@ pub enum CodebaseIndexManagerEvent {
retrieval_id: RetrievalID,
error_message: String,
},
SyncStateUpdated,
SyncStateUpdated {
root_path: PathBuf,
},
IndexMetadataUpdated {
root_path: PathBuf,
event: WorkspaceMetadataEvent,
@@ -87,7 +95,9 @@ pub enum CodebaseIndexManagerEvent {
RemoveExpiredIndexMetadata {
expired_metadata: Arc<Vec<PathBuf>>,
},
NewIndexCreated,
NewIndexCreated {
root_path: PathBuf,
},
}
/// User-facing indexing errors.
@@ -132,6 +142,7 @@ pub struct CodebaseIndexStatus {
pub(super) has_synced_version: bool,
pub(super) last_sync_successful: Option<CodebaseIndexFinishedStatus>,
pub(super) sync_progress: Option<SyncProgress>,
pub(super) root_hash: Option<NodeHash>,
}
impl CodebaseIndexStatus {
@@ -156,17 +167,127 @@ impl CodebaseIndexStatus {
pub fn sync_progress(&self) -> Option<&SyncProgress> {
self.sync_progress.as_ref()
}
pub fn root_hash(&self) -> Option<&NodeHash> {
self.root_hash.as_ref()
}
}
#[derive(Debug, Eq, PartialEq)]
struct CodebaseIndexStatusEventKey {
has_pending: bool,
has_synced_version: bool,
last_sync_status: Option<CodebaseIndexFinishedStatusEventKey>,
sync_progress: Option<SyncProgressEventKey>,
root_hash: Option<String>,
}
impl From<&CodebaseIndexStatus> for CodebaseIndexStatusEventKey {
fn from(status: &CodebaseIndexStatus) -> Self {
Self {
has_pending: status.has_pending,
has_synced_version: status.has_synced_version,
last_sync_status: status
.last_sync_successful
.as_ref()
.map(CodebaseIndexFinishedStatusEventKey::from),
sync_progress: status
.sync_progress
.as_ref()
.map(SyncProgressEventKey::from),
root_hash: status.root_hash.as_ref().map(ToString::to_string),
}
}
}
#[derive(Debug, Eq, PartialEq)]
enum CodebaseIndexFinishedStatusEventKey {
Completed,
Failed(String),
}
impl From<&CodebaseIndexFinishedStatus> for CodebaseIndexFinishedStatusEventKey {
fn from(status: &CodebaseIndexFinishedStatus) -> Self {
match status {
CodebaseIndexFinishedStatus::Completed => Self::Completed,
CodebaseIndexFinishedStatus::Failed(error) => Self::Failed(error.to_string()),
}
}
}
#[derive(Debug, Eq, PartialEq)]
enum SyncProgressEventKey {
Discovering {
total_nodes: usize,
},
Syncing {
completed_nodes: usize,
total_nodes: usize,
},
}
impl From<&SyncProgress> for SyncProgressEventKey {
fn from(progress: &SyncProgress) -> Self {
match progress {
SyncProgress::Discovering { total_nodes } => Self::Discovering {
total_nodes: *total_nodes,
},
SyncProgress::Syncing {
completed_nodes,
total_nodes,
} => Self::Syncing {
completed_nodes: *completed_nodes,
total_nodes: *total_nodes,
},
}
}
}
pub enum BuildSource<'a> {
FromPath(&'a Path),
FromPersistedMetadata(WorkspaceMetadata),
}
pub struct CodebaseIndexManagerConfig {
persisted_index_metadata: Vec<WorkspaceMetadata>,
max_index_count: Option<usize>,
max_files_repo_limit: usize,
embedding_generation_batch_size: usize,
store_client: Arc<dyn StoreClient>,
indexing_enabled: bool,
restore_persisted_indices_on_startup: bool,
}
impl CodebaseIndexManagerConfig {
pub fn new(
persisted_index_metadata: Vec<WorkspaceMetadata>,
max_index_count: Option<usize>,
max_files_repo_limit: usize,
embedding_generation_batch_size: usize,
store_client: Arc<dyn StoreClient>,
indexing_enabled: bool,
) -> Self {
Self {
persisted_index_metadata,
max_index_count,
max_files_repo_limit,
embedding_generation_batch_size,
store_client,
indexing_enabled,
restore_persisted_indices_on_startup: true,
}
}
pub fn defer_persisted_index_restore(mut self) -> Self {
self.restore_persisted_indices_on_startup = false;
self
}
}
/// Manager for the codebase index states across the app.
pub struct CodebaseIndexManager {
codebase_indices: HashMap<PathBuf, ModelHandle<CodebaseIndex>>,
last_emitted_codebase_index_statuses: HashMap<PathBuf, CodebaseIndexStatusEventKey>,
store_client: Arc<dyn StoreClient>,
#[cfg(feature = "local_fs")]
@@ -179,6 +300,11 @@ pub struct CodebaseIndexManager {
max_files_repo_limit: usize,
embedding_generation_batch_size: usize,
indexing_enabled: bool,
#[cfg(feature = "local_fs")]
snapshot_storage: Option<SnapshotStorage>,
}
impl CodebaseIndexManager {
@@ -189,25 +315,100 @@ impl CodebaseIndexManager {
max_files_repo_limit: usize,
embedding_generation_batch_size: usize,
store_client: Arc<dyn StoreClient>,
indexing_enabled: bool,
ctx: &mut ModelContext<Self>,
) -> Self {
let config = CodebaseIndexManagerConfig::new(
persisted_index_metadata,
max_index_count,
max_files_repo_limit,
embedding_generation_batch_size,
store_client,
indexing_enabled,
);
Self::new_with_config(config, ctx)
}
#[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
pub fn new_with_config(
config: CodebaseIndexManagerConfig,
ctx: &mut ModelContext<Self>,
) -> Self {
#[cfg(feature = "local_fs")]
{
report_if_error!(migrate_snapshots_to_secure_dir_if_needed());
Self::new_with_snapshot_storage(config, SnapshotStorage::app_default(), ctx)
}
#[cfg(not(feature = "local_fs"))]
{
Self::new_internal(config, ctx)
}
}
#[cfg(feature = "local_fs")]
pub fn new_with_snapshot_storage(
config: CodebaseIndexManagerConfig,
snapshot_storage: Option<SnapshotStorage>,
ctx: &mut ModelContext<Self>,
) -> Self {
Self::new_internal(config, snapshot_storage, ctx)
}
#[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
fn new_internal(
config: CodebaseIndexManagerConfig,
#[cfg(feature = "local_fs")] snapshot_storage: Option<SnapshotStorage>,
ctx: &mut ModelContext<Self>,
) -> Self {
let CodebaseIndexManagerConfig {
persisted_index_metadata,
max_index_count,
max_files_repo_limit,
embedding_generation_batch_size,
store_client,
indexing_enabled,
restore_persisted_indices_on_startup,
} = config;
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
let file_watcher = ctx.add_model(|ctx| BulkFilesystemWatcher::new(REPO_WATCHER_DEBOUNCE_DURATION, ctx));
ctx.subscribe_to_model(&file_watcher, Self::handle_watcher_event);
}
}
if !indexing_enabled {
log::debug!(
"Codebase indexing disabled for this launch mode; skipping restore of {:?} persisted codebase indices",
persisted_index_metadata.len()
);
return Self {
codebase_indices: HashMap::new(),
last_emitted_codebase_index_statuses: HashMap::new(),
store_client,
#[cfg(feature = "local_fs")]
watcher: file_watcher,
build_queue: BuildQueue::empty(),
max_indices: max_index_count,
max_files_repo_limit,
embedding_generation_batch_size,
indexing_enabled,
#[cfg(feature = "local_fs")]
snapshot_storage,
};
}
log::debug!(
"Received {:?} persisted codebase indices",
persisted_index_metadata.len()
);
#[cfg(feature = "local_fs")]
report_if_error!(migrate_snapshots_to_secure_dir_if_needed());
let (invalid_metadata, valid_metadata) =
split_snapshot_metadata_by_validity(persisted_index_metadata);
let (invalid_metadata, valid_metadata) = split_snapshot_metadata_by_validity(
persisted_index_metadata,
snapshot_storage.as_ref(),
);
#[cfg(not(feature = "local_fs"))]
let (invalid_metadata, valid_metadata) = (persisted_index_metadata, Vec::new());
ctx.emit(CodebaseIndexManagerEvent::RemoveExpiredIndexMetadata {
expired_metadata: Arc::new(
@@ -217,16 +418,18 @@ impl CodebaseIndexManager {
.collect(),
),
});
if let Some(snapshot_file_dir) = snapshot_dir() {
clean_up_snapshot_files(&snapshot_file_dir, &valid_metadata);
#[cfg(feature = "local_fs")]
if let Some(snapshot_storage) = snapshot_storage.as_ref() {
clean_up_snapshot_files(snapshot_storage.path(), &valid_metadata);
}
// For the moment, we've decided to load all snapshots regardless of the index count.
let build_queue = BuildQueue::new_with_persisted(valid_metadata);
let build_queue =
BuildQueue::new_with_persisted(valid_metadata, restore_persisted_indices_on_startup);
let mut me = Self {
codebase_indices: HashMap::new(),
last_emitted_codebase_index_statuses: HashMap::new(),
store_client,
#[cfg(feature = "local_fs")]
watcher: file_watcher,
@@ -234,12 +437,12 @@ impl CodebaseIndexManager {
max_indices: max_index_count,
max_files_repo_limit,
embedding_generation_batch_size,
indexing_enabled,
#[cfg(feature = "local_fs")]
snapshot_storage,
};
// Start building the first index in the queue.
if let Some(next_repo) = me.build_queue.pick_next_sync() {
me.build_and_sync_codebase_index(BuildSource::FromPersistedMetadata(next_repo), ctx);
}
me.start_next_queued_index(ctx);
me
}
@@ -250,6 +453,7 @@ impl CodebaseIndexManager {
let file_watcher = ctx.add_model(|_| BulkFilesystemWatcher::new_for_test());
Self {
codebase_indices: HashMap::new(),
last_emitted_codebase_index_statuses: HashMap::new(),
store_client,
#[cfg(feature = "local_fs")]
watcher: file_watcher,
@@ -257,6 +461,9 @@ impl CodebaseIndexManager {
max_indices: None,
max_files_repo_limit: 0,
embedding_generation_batch_size: 100,
indexing_enabled: true,
#[cfg(feature = "local_fs")]
snapshot_storage: SnapshotStorage::app_default(),
}
}
@@ -306,8 +513,15 @@ impl CodebaseIndexManager {
// Remove snapshots from disk.
let to_drop_clone = to_drop.clone();
#[cfg(feature = "local_fs")]
let snapshot_storage = self.snapshot_storage.clone();
ctx.spawn(
async move { Self::drop_index_snapshots(to_drop_clone).await },
async move {
#[cfg(feature = "local_fs")]
Self::drop_index_snapshots(snapshot_storage, to_drop_clone).await;
#[cfg(not(feature = "local_fs"))]
let _ = to_drop_clone;
},
|_, _, _| {},
);
@@ -317,11 +531,15 @@ impl CodebaseIndexManager {
});
}
/// Remove the gien index snapshots from disk.
async fn drop_index_snapshots(to_drop: Vec<PathBuf>) {
if let Some(snapshot_dir) = snapshot_dir() {
/// Remove the given index snapshots from disk.
#[cfg(feature = "local_fs")]
async fn drop_index_snapshots(
snapshot_storage: Option<SnapshotStorage>,
to_drop: Vec<PathBuf>,
) {
if let Some(snapshot_storage) = snapshot_storage {
for codebase_root in &to_drop {
Self::drop_index_snapshot(&snapshot_dir, codebase_root).await;
Self::drop_index_snapshot(snapshot_storage.path(), codebase_root).await;
}
}
}
@@ -345,6 +563,7 @@ impl CodebaseIndexManager {
// Drop the in-memory index.
self.codebase_indices.remove(root_path);
self.last_emitted_codebase_index_statuses.remove(root_path);
// Stop the filewatcher from receiving events for this codebase.
#[cfg(feature = "local_fs")]
@@ -359,11 +578,16 @@ impl CodebaseIndexManager {
// Remove snapshot from disk.
let root_path_clone = root_path.clone();
#[cfg(feature = "local_fs")]
let snapshot_storage = self.snapshot_storage.clone();
ctx.spawn(
async move {
if let Some(snapshot_dir) = snapshot_dir() {
Self::drop_index_snapshot(&snapshot_dir, &root_path_clone).await;
#[cfg(feature = "local_fs")]
if let Some(snapshot_storage) = snapshot_storage {
Self::drop_index_snapshot(snapshot_storage.path(), &root_path_clone).await;
}
#[cfg(not(feature = "local_fs"))]
let _ = root_path_clone;
},
|_, _, _| {},
);
@@ -451,6 +675,7 @@ impl CodebaseIndexManager {
#[cfg(feature = "local_fs")]
fn handle_watcher_event(
&mut self,
_: ModelHandle<BulkFilesystemWatcher>,
event: &BulkFilesystemWatcherEvent,
ctx: &mut ModelContext<Self>,
) {
@@ -462,6 +687,9 @@ impl CodebaseIndexManager {
}
pub fn handle_active_session_changed(&mut self, active_directory: &Path) {
if !self.is_indexing_enabled() {
return;
}
let Some(root_path) = self.root_path_for_codebase(active_directory) else {
return;
};
@@ -517,11 +745,17 @@ impl CodebaseIndexManager {
/// Ensures the current number of indices is below the maximum.
pub fn can_create_new_indices(&self) -> bool {
if !self.is_indexing_enabled() {
return false;
}
self.max_indices
.is_none_or(|max_indices| self.codebase_indices.len() < max_indices)
}
pub fn handle_session_bootstrapped(&mut self, working_directory: &Path) {
if !self.is_indexing_enabled() {
return;
}
let Some(root_path) = self.root_path_for_codebase(working_directory) else {
return;
};
@@ -553,6 +787,22 @@ impl CodebaseIndexManager {
})
}
pub fn fragment_metadatas_from_hashes(
&self,
repo_path: &Path,
root_hash: &NodeHash,
content_hashes: &[ContentHash],
app: &AppContext,
) -> Result<HashMap<ContentHash, Vec<FragmentMetadata>>, FragmentMetadataLookupError> {
let (codebase_index, _) = self
.get_codebase_index_internal(repo_path)
.map_err(|_| FragmentMetadataLookupError::IndexNotFound)?;
codebase_index
.as_ref(app)
.fragment_metadatas_from_hashes(root_hash, content_hashes)
}
pub fn get_codebase_paths(&self) -> impl Iterator<Item = &PathBuf> {
self.codebase_indices.keys()
}
@@ -561,13 +811,37 @@ impl CodebaseIndexManager {
self.codebase_indices.len()
}
pub fn index_directory(&mut self, directory: PathBuf, ctx: &mut ModelContext<Self>) {
let directory = dunce::canonicalize(&directory).unwrap_or(directory);
if !self.codebase_indices.contains_key(&directory) {
self.build_and_sync_codebase_index(BuildSource::FromPath(&directory), ctx);
// Starting a new codebase index should be considered into sync state updates.
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated);
pub fn is_indexing_enabled(&self) -> bool {
self.indexing_enabled
}
pub fn start_persisted_index_restore(&mut self, ctx: &mut ModelContext<Self>) {
if !self.is_indexing_enabled() {
return;
}
if self.build_queue.start() {
self.start_next_queued_index(ctx);
}
}
pub fn index_directory(&mut self, directory: PathBuf, ctx: &mut ModelContext<Self>) -> bool {
if !self.is_indexing_enabled() {
return false;
}
if self.root_path_for_codebase(&directory).is_none() {
if !self.build_and_sync_codebase_index(BuildSource::FromPath(&directory), ctx) {
return false;
}
let indexed_directory = self
.root_path_for_codebase(&directory)
.unwrap_or_else(|| directory.clone());
self.record_codebase_index_status(&indexed_directory, ctx);
// Starting a new codebase index should be considered into sync state updates.
ctx.emit(CodebaseIndexManagerEvent::NewIndexCreated {
root_path: indexed_directory,
});
}
true
}
#[cfg(feature = "local_fs")]
@@ -577,9 +851,21 @@ impl CodebaseIndexManager {
gitignores: Arc<Vec<Gitignore>>,
ctx: &mut ModelContext<Self>,
) {
let watch_filter = WatchFilter::with_filter(Arc::new(move |path| {
path_passes_filters(path, gitignores.as_slice())
}));
// The codebase indexer only cares about source files:
// skip anything inside `.git/` and anything matched by gitignore
// (including descendants of an ignored ancestor directory).
// The same predicate gates both directory descent and event emission.
let filter = Arc::new(move |path: &Path| {
!is_git_internal_path(path)
&& !matches_gitignores(
path,
path.is_dir(),
gitignores.as_slice(),
true, /* check_ancestors */
)
});
let watch_filter = WatchFilter::with_filter(filter.clone(), filter);
self.watcher.update(ctx, |watcher, _ctx| {
std::mem::drop(watcher.register_path(
root_path,
@@ -607,9 +893,12 @@ impl CodebaseIndexManager {
&mut self,
build_source: BuildSource,
ctx: &mut ModelContext<Self>,
) {
) -> bool {
if !self.is_indexing_enabled() {
return false;
}
if !self.can_create_new_indices() {
return;
return false;
}
let repo_path = match build_source {
@@ -624,7 +913,7 @@ impl CodebaseIndexManager {
Ok(path) => path,
Err(e) => {
log::error!("Failed to canonicalize repository path: {e:?}");
return;
return false;
}
};
@@ -635,7 +924,7 @@ impl CodebaseIndexManager {
Ok(handle) => handle,
Err(e) => {
log::error!("Failed to start tracking repository: {e:?}");
return;
return false;
}
};
@@ -646,11 +935,15 @@ impl CodebaseIndexManager {
.codebase_indices
.entry(canonical_key)
.or_insert_with(|| {
#[cfg(feature = "local_fs")]
let snapshot_storage = self.snapshot_storage.clone();
let index = Self::build_and_sync_codebase_index_internal(
self.store_client.clone(),
handle,
self.max_files_repo_limit,
self.embedding_generation_batch_size,
#[cfg(feature = "local_fs")]
snapshot_storage,
ctx,
);
@@ -666,6 +959,7 @@ impl CodebaseIndexManager {
index.update_timestamps_from_metadata(metadata);
});
}
true
}
/// Checks whether a snapshot exists for the index and attempts to load it;
@@ -675,6 +969,7 @@ impl CodebaseIndexManager {
repository: ModelHandle<Repository>,
max_files_repo_limit: usize,
embedding_generation_batch_size: usize,
#[cfg(feature = "local_fs")] snapshot_storage: Option<SnapshotStorage>,
ctx: &mut ModelContext<Self>,
) -> ModelHandle<CodebaseIndex> {
let codebase_index = ctx.add_model(|ctx| {
@@ -684,13 +979,17 @@ impl CodebaseIndexManager {
.as_ref(ctx)
.root_dir()
.to_local_path()
.is_some_and(|p| has_snapshot(&p))
.is_some_and(|p| {
snapshot_storage
.as_ref()
.is_some_and(|storage| storage.has_snapshot(&p))
})
{
if let Some(snapshot_dir) = snapshot_dir() {
if let Some(snapshot_storage) = snapshot_storage.as_ref() {
let read_snapshot_start_time = Instant::now();
match read_snapshot(
store_client.clone(),
snapshot_dir.as_path(),
snapshot_storage.path(),
repository.clone(),
max_files_repo_limit,
embedding_generation_batch_size,
@@ -733,6 +1032,7 @@ impl CodebaseIndexManager {
fn handle_codebase_index_event(
&mut self,
_: ModelHandle<CodebaseIndex>,
event: &CodebaseIndexEvent,
ctx: &mut ModelContext<Self>,
) {
@@ -753,8 +1053,8 @@ impl CodebaseIndexManager {
fragments: fragments.clone(),
out_of_sync_delay: *out_of_sync_delay,
}),
CodebaseIndexEvent::SyncStateUpdated => {
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated)
CodebaseIndexEvent::SyncStateUpdated { root_path } => {
self.maybe_emit_sync_state_updated(root_path, ctx);
}
CodebaseIndexEvent::IndexMetadataUpdated { root_path, event } => {
ctx.emit(CodebaseIndexManagerEvent::IndexMetadataUpdated {
@@ -785,11 +1085,42 @@ impl CodebaseIndexManager {
}
}
fn maybe_emit_sync_state_updated(&mut self, root_path: &Path, ctx: &mut ModelContext<Self>) {
if self.record_codebase_index_status(root_path, ctx) {
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated {
root_path: root_path.to_path_buf(),
});
}
}
fn record_codebase_index_status(
&mut self,
root_path: &Path,
ctx: &mut ModelContext<Self>,
) -> bool {
let root_path = dunce::canonicalize(root_path).unwrap_or_else(|_| root_path.to_path_buf());
let Some(status) = self.get_codebase_index_status_for_path(root_path.as_path(), ctx) else {
return false;
};
let key = CodebaseIndexStatusEventKey::from(&status);
match self.last_emitted_codebase_index_statuses.get(&root_path) {
Some(previous_key) if previous_key == &key => false,
Some(_) | None => {
self.last_emitted_codebase_index_statuses
.insert(root_path, key);
true
}
}
}
fn on_index_build_finished(&mut self, finished_repo: &Path, ctx: &mut ModelContext<Self>) {
let Ok(_) = self.get_codebase_index_internal(finished_repo) else {
return;
};
self.start_next_queued_index(ctx);
}
fn start_next_queued_index(&mut self, ctx: &mut ModelContext<Self>) {
if let Some(next_repo) = self.build_queue.pick_next_sync() {
self.build_and_sync_codebase_index(BuildSource::FromPersistedMetadata(next_repo), ctx);
}
@@ -810,6 +1141,20 @@ impl CodebaseIndexManager {
.ok()
}
pub fn with_indexed_codebase<T>(
&mut self,
path: &Path,
on_found: impl FnOnce(&mut Self, &Path, &mut ModelContext<Self>) -> T,
on_missing: impl FnOnce(&mut Self, &Path, &mut ModelContext<Self>) -> T,
ctx: &mut ModelContext<Self>,
) -> T {
let Some(indexed_repo_path) = self.root_path_for_codebase(path) else {
return on_missing(self, path, ctx);
};
on_found(self, indexed_repo_path.as_path(), ctx)
}
fn get_codebase_index_internal(
&self,
path: &Path,
@@ -909,15 +1254,15 @@ impl CodebaseIndexManager {
}
};
let snapshot_dir = match snapshot_dir() {
Some(dir) => dir,
let snapshot_storage = match self.snapshot_storage.as_ref() {
Some(storage) => storage,
None => {
log::warn!("No snapshot directory to write to");
Self::schedule_next_snapshot_write(repo_path, ctx);
return;
}
};
let snapshot_path = snapshot_path(&snapshot_dir, repo_path.as_path());
let snapshot_path = snapshot_storage.snapshot_path(repo_path.as_path());
// Update timestamp eagerly so concurrent calls to has_unsnapshotted_changes()
// won't trigger a duplicate snapshot while the background write is in progress.
@@ -974,6 +1319,9 @@ impl CodebaseIndexManager {
directory_path: &Path,
ctx: &mut ModelContext<Self>,
) -> anyhow::Result<()> {
if !self.is_indexing_enabled() {
return Ok(());
}
// Find the root path for this directory's codebase
let Some(repo_path) = self.root_path_for_codebase(directory_path) else {
return Err(anyhow::anyhow!("Failed to find root path for directory"));
@@ -987,7 +1335,7 @@ impl CodebaseIndexManager {
codebase_index.update(ctx, |index, _ctx| {
// Check if the index is in a state where it can perform incremental updates
let status = index.codebase_index_status();
if status.has_pending {
if status.last_sync_successful() != Some(true) {
return;
}
@@ -1013,3 +1361,7 @@ impl Entity for CodebaseIndexManager {
}
impl SingletonEntity for CodebaseIndexManager {}
#[cfg(test)]
#[path = "manager_tests.rs"]
mod tests;
@@ -0,0 +1,379 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
#[cfg(feature = "local_fs")]
use chrono::Utc;
#[cfg(feature = "local_fs")]
use repo_metadata::DirectoryWatcher;
use galaxyui_core::App;
use super::{
BuildSource, CodebaseIndexFinishedStatus, CodebaseIndexManager, CodebaseIndexManagerConfig,
CodebaseIndexStatus, CodebaseIndexStatusEventKey, CodebaseIndexingError, SyncProgress,
};
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
#[cfg(feature = "local_fs")]
use crate::index::full_source_code_embedding::SnapshotStorage;
use crate::workspace::WorkspaceMetadata;
fn workspace_metadata(path: impl Into<PathBuf>) -> WorkspaceMetadata {
WorkspaceMetadata {
path: path.into(),
navigated_ts: None,
modified_ts: None,
queried_ts: None,
}
}
fn codebase_index_status(
has_pending: bool,
has_synced_version: bool,
last_sync_successful: Option<CodebaseIndexFinishedStatus>,
sync_progress: Option<SyncProgress>,
) -> CodebaseIndexStatus {
CodebaseIndexStatus {
has_pending,
has_synced_version,
last_sync_successful,
sync_progress,
root_hash: None,
}
}
#[test]
fn codebase_index_status_event_key_matches_identical_statuses() {
let first_status = codebase_index_status(
true,
true,
None,
Some(SyncProgress::Syncing {
completed_nodes: 1,
total_nodes: 2,
}),
);
let duplicate_status = codebase_index_status(
true,
true,
None,
Some(SyncProgress::Syncing {
completed_nodes: 1,
total_nodes: 2,
}),
);
assert_eq!(
CodebaseIndexStatusEventKey::from(&first_status),
CodebaseIndexStatusEventKey::from(&duplicate_status)
);
}
#[test]
fn codebase_index_status_event_key_detects_semantic_changes() {
let syncing_status = codebase_index_status(
true,
true,
None,
Some(SyncProgress::Syncing {
completed_nodes: 1,
total_nodes: 2,
}),
);
let completed_status = codebase_index_status(
false,
true,
Some(CodebaseIndexFinishedStatus::Completed),
None,
);
let failed_status = codebase_index_status(
false,
true,
Some(CodebaseIndexFinishedStatus::Failed(
CodebaseIndexingError::BuildTreeError,
)),
None,
);
assert_ne!(
CodebaseIndexStatusEventKey::from(&syncing_status),
CodebaseIndexStatusEventKey::from(&completed_status)
);
assert_ne!(
CodebaseIndexStatusEventKey::from(&completed_status),
CodebaseIndexStatusEventKey::from(&failed_status)
);
}
#[test]
fn initializes_with_indexing_enabled_when_configured() {
App::test((), |app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
vec![workspace_metadata("repo")],
Some(1),
1000,
32,
Arc::new(MockStoreClient),
true,
ctx,
)
});
manager.read(&app, |manager, _| {
assert!(manager.is_indexing_enabled());
assert_eq!(manager.num_active_indices(), 0);
assert!(manager.can_create_new_indices());
});
});
}
#[test]
fn initializes_with_indexing_disabled_when_configured() {
App::test((), |app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
vec![workspace_metadata("repo")],
Some(1),
1000,
32,
Arc::new(MockStoreClient),
false,
ctx,
)
});
manager.read(&app, |manager, _| {
assert!(!manager.is_indexing_enabled());
assert_eq!(manager.num_active_indices(), 0);
assert!(!manager.can_create_new_indices());
});
});
}
#[test]
#[cfg(feature = "local_fs")]
fn initializes_with_injected_snapshot_storage_when_configured() {
App::test((), |app| async move {
let snapshot_dir = tempfile::tempdir().unwrap();
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
let expected_snapshot_dir = storage.path().to_path_buf();
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new_with_snapshot_storage(
CodebaseIndexManagerConfig::new(
vec![workspace_metadata("repo")],
Some(1),
1000,
32,
Arc::new(MockStoreClient),
false,
),
Some(storage),
ctx,
)
});
manager.read(&app, |manager, _| {
let snapshot_storage = manager.snapshot_storage.as_ref().unwrap();
assert_eq!(snapshot_storage.path(), expected_snapshot_dir);
assert!(!snapshot_storage.is_app_default());
});
});
}
#[test]
fn persisted_index_restore_starts_on_startup_by_default() {
App::test((), |app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
true,
ctx,
)
});
manager.read(&app, |manager, _| {
assert!(manager.build_queue.is_running());
});
});
}
#[test]
#[cfg(feature = "local_fs")]
fn deferred_persisted_index_restore_starts_once() {
App::test((), |mut app| async move {
app.add_singleton_model(DirectoryWatcher::new);
let snapshot_dir = tempfile::tempdir().unwrap();
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
let first_repo = tempfile::tempdir().unwrap();
let second_repo = tempfile::tempdir().unwrap();
let mut first_metadata = workspace_metadata(first_repo.path());
first_metadata.modified_ts = Some(Utc::now());
let mut second_metadata = workspace_metadata(second_repo.path());
second_metadata.modified_ts = Some(Utc::now());
std::fs::write(storage.snapshot_path(first_repo.path()), b"snapshot").unwrap();
std::fs::write(storage.snapshot_path(second_repo.path()), b"snapshot").unwrap();
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new_with_snapshot_storage(
CodebaseIndexManagerConfig::new(
vec![first_metadata, second_metadata],
Some(2),
1000,
32,
Arc::new(MockStoreClient),
true,
)
.defer_persisted_index_restore(),
Some(storage),
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
assert!(!manager.build_queue.is_running());
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 2);
manager.start_persisted_index_restore(ctx);
assert!(manager.build_queue.is_running());
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 1);
manager.start_persisted_index_restore(ctx);
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 1);
});
});
}
#[test]
fn can_create_new_indices_honors_max_limit_when_enabled() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
true,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
assert!(manager.can_create_new_indices());
manager.update_max_limits(Some(0), 1000, 32, ctx);
assert!(!manager.can_create_new_indices());
});
});
}
#[test]
fn index_directory_is_noop_when_indexing_disabled() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
false,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
assert!(!manager.index_directory(PathBuf::from("repo"), ctx));
assert_eq!(manager.num_active_indices(), 0);
});
});
}
#[test]
fn index_directory_reports_when_max_index_limit_prevents_creation() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(0),
1000,
32,
Arc::new(MockStoreClient),
true,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
assert!(!manager.index_directory(PathBuf::from("repo"), ctx));
assert_eq!(manager.num_active_indices(), 0);
});
});
}
#[test]
fn build_and_sync_is_noop_when_indexing_disabled() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
false,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
assert!(!manager
.build_and_sync_codebase_index(BuildSource::FromPath(Path::new("repo")), ctx));
assert_eq!(manager.num_active_indices(), 0);
});
});
}
#[test]
fn trigger_incremental_sync_returns_err_when_enabled_and_index_missing() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
true,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
let result = manager.trigger_incremental_sync_for_path(Path::new("repo"), ctx);
assert!(result.is_err());
});
});
}
#[test]
fn trigger_incremental_sync_returns_ok_when_indexing_disabled() {
App::test((), |mut app| async move {
let manager = app.add_singleton_model(|ctx| {
CodebaseIndexManager::new(
Vec::new(),
Some(1),
1000,
32,
Arc::new(MockStoreClient),
false,
ctx,
)
});
manager.update(&mut app, |manager, ctx| {
let result = manager.trigger_incremental_sync_for_path(Path::new("repo"), ctx);
assert!(result.is_ok());
});
});
}
@@ -1,13 +1,16 @@
//! Common types for hashes that identify codebase embedding state.
use std::fmt;
use std::str::FromStr;
use std::sync::Arc;
use generic_array::GenericArray;
use serde::{Deserialize, Serialize};
use sha2::{digest::OutputSizeUser, Digest, Sha256};
use std::{fmt, str::FromStr, sync::Arc};
use crate::index::full_source_code_embedding::chunker::Fragment;
use sha2::digest::OutputSizeUser;
use sha2::{Digest, Sha256};
use super::Error;
use crate::index::full_source_code_embedding::chunker::Fragment;
/// The hash of an *intermediate* node in the [`MerkleTree`].
///
@@ -223,5 +226,5 @@ impl fmt::Display for MerkleHash {
}
#[cfg(test)]
#[path = "hash_test.rs"]
#[path = "hash_tests.rs"]
mod hash_test;
@@ -1,6 +1,7 @@
use std::path::{Path, PathBuf};
use super::MerkleHash;
use crate::index::full_source_code_embedding::chunker::Fragment;
use std::path::{Path, PathBuf};
#[test]
fn test_fragment_hash_from_content() {
@@ -1,4 +1,5 @@
use super::{chunker::Fragment, Error};
use super::chunker::Fragment;
use super::Error;
mod hash;
mod node;
@@ -1,8 +1,8 @@
use crate::index::{
THREADPOOL, {DirectoryEntry, Entry, FileMetadata},
};
use anyhow::anyhow;
use std::collections::{HashMap, HashSet};
use std::ops::Range;
use std::path::{Path, PathBuf};
use anyhow::anyhow;
use chrono::{DateTime, Utc};
use galaxy_util::standardized_path::StandardizedPath;
use itertools::Itertools;
@@ -10,26 +10,18 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
use repo_metadata::entry::is_file_parsable;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::{HashMap, HashSet},
ops::Range,
path::{Path, PathBuf},
};
use string_offset::ByteOffset;
use crate::index::full_source_code_embedding::{
chunker::chunk_code,
fragment_metadata::{FragmentMetadata, LeafToFragmentMetadataUpdates},
Error,
};
use super::{
hash::MerkleHash,
serialized_tree::{SerializedFilesystemInfo, SerializedMerkleNode},
tree::UpdateFileResult,
ContentHash, DirEntryOrFragment, NodeHash,
use super::hash::MerkleHash;
use super::serialized_tree::{SerializedFilesystemInfo, SerializedMerkleNode};
use super::tree::UpdateFileResult;
use super::{ContentHash, DirEntryOrFragment, NodeHash};
use crate::index::full_source_code_embedding::chunker::chunk_code;
use crate::index::full_source_code_embedding::fragment_metadata::{
FragmentMetadata, LeafToFragmentMetadataUpdates,
};
use crate::index::full_source_code_embedding::Error;
use crate::index::{DirectoryEntry, Entry, FileMetadata, THREADPOOL};
/// ID that uniquely identifies a node in the merkle tree. It contains the node type
/// as well as metadata that distinguishes nodes of the same type.
@@ -707,5 +699,5 @@ impl NodeMask {
}
#[cfg(test)]
#[path = "node_test.rs"]
#[path = "node_tests.rs"]
mod tests;
@@ -1,12 +1,11 @@
use crate::index::full_source_code_embedding::{
fragment_metadata::LeafToFragmentMetadataUpdates, merkle_tree::DirEntryOrFragment,
};
use std::collections::HashSet;
use repo_metadata::{DirectoryEntry, Entry};
use virtual_fs::{Stub, VirtualFS};
use std::collections::HashSet;
use super::{MerkleNode, NodeMask};
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadataUpdates;
use crate::index::full_source_code_embedding::merkle_tree::DirEntryOrFragment;
/// Tests that node hashes for directories are sorted (meaning they are resilient to files within
/// the directory being in a different order).
@@ -1,18 +1,17 @@
use std::{
ops::Range,
path::{Path, PathBuf},
};
use super::{hash::MerkleHash, node::NodeId, MerkleTree, NodeHash, NodeLens};
use crate::index::full_source_code_embedding::fragment_metadata::{
FragmentLocation, LeafToFragmentMetadata,
};
use std::ops::Range;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use string_offset::ByteOffset;
use super::hash::MerkleHash;
use super::node::NodeId;
use super::{MerkleTree, NodeHash, NodeLens};
use crate::index::full_source_code_embedding::fragment_metadata::{
FragmentLocation, LeafToFragmentMetadata,
};
#[derive(Debug, PartialEq, Serialize, Deserialize)]
pub(crate) struct SerializedCodebaseIndex {
tree: SerializedMerkleTree,
@@ -196,5 +195,5 @@ impl SerializedMerkleNode {
}
#[cfg(test)]
#[path = "serialized_tree_test.rs"]
#[path = "serialized_tree_tests.rs"]
mod tests;
@@ -2,12 +2,11 @@ use futures::executor::block_on;
use serde_json;
use virtual_fs::VirtualFS;
use super::SerializedCodebaseIndex;
use crate::index::full_source_code_embedding::merkle_tree::{
construct_test_merkle_tree, MerkleTree,
};
use super::SerializedCodebaseIndex;
#[test]
fn round_trip_index_serialize_deserialize_json() {
VirtualFS::test("test_nodes_from_path_json", |dirs, mut sandbox| {
@@ -1,8 +1,9 @@
use super::MerkleTree;
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadata;
use repo_metadata::{DirectoryEntry, Entry};
use virtual_fs::{Stub, VirtualFS};
use super::MerkleTree;
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadata;
/// Construct a test Merkle tree with the following structure:
/// ```
/// root.txt
@@ -1,21 +1,17 @@
use crate::index::Entry;
use std::collections::{HashSet, VecDeque};
use std::path::PathBuf;
use anyhow::anyhow;
use cfg_if::cfg_if;
use std::{
collections::{HashSet, VecDeque},
path::PathBuf,
};
use super::node::{ChildrenPath, MerkleNode, NodeLens, NodeMask};
use super::serialized_tree::SerializedMerkleTree;
use super::DirEntryOrFragment;
use crate::index::full_source_code_embedding::fragment_metadata::{
LeafToFragmentMetadata, LeafToFragmentMetadataUpdates,
};
use crate::index::full_source_code_embedding::Error;
use super::{
node::{ChildrenPath, MerkleNode, NodeLens, NodeMask},
serialized_tree::SerializedMerkleTree,
DirEntryOrFragment,
};
use crate::index::Entry;
pub(super) enum UpdateFileResult {
Deleted,
@@ -210,5 +206,5 @@ impl MerkleTree {
}
#[cfg(test)]
#[path = "tree_test.rs"]
#[path = "tree_tests.rs"]
mod tests;
@@ -1,11 +1,9 @@
use crate::index::full_source_code_embedding::merkle_tree::{
construct_test_merkle_tree, node::ChildrenPath,
};
use futures::executor::block_on;
use virtual_fs::VirtualFS;
use super::*;
use crate::index::full_source_code_embedding::merkle_tree::construct_test_merkle_tree;
use crate::index::full_source_code_embedding::merkle_tree::node::ChildrenPath;
#[test]
fn test_nodes_from_path() {
@@ -5,19 +5,21 @@ mod fragment_metadata;
pub mod manager;
mod merkle_tree;
mod priority_queue;
pub mod search_shaping;
mod snapshot;
pub mod store_client;
mod sync_client;
use std::{ops::Range, path::PathBuf, time::Duration};
pub use sync_client::SyncTask;
use std::ops::Range;
use std::path::{Path, PathBuf};
use std::time::Duration;
pub use codebase_index::{CodebaseIndex, RetrievalID, SyncProgress};
pub use fragment_metadata::{FragmentLocation as FragmentMetadataLocation, FragmentMetadata};
pub use merkle_tree::{ContentHash, NodeHash};
use fragment_metadata::FragmentMetadata;
use galaxy_graphql::queries::rerank_fragments::FragmentLocationInput;
pub use snapshot::SnapshotStorage;
use string_offset::ByteOffset;
pub use sync_client::SyncTask;
use thiserror::Error;
#[derive(Error, Debug)]
@@ -87,6 +89,7 @@ pub enum EmbeddingConfig {
Voyage3_5_Lite_512,
#[default]
Voyage3_5_512,
Voyage4_512,
}
#[derive(Debug, Clone)]
@@ -115,6 +118,9 @@ impl From<EmbeddingConfig> for galaxy_graphql::full_source_code_embedding::Embed
EmbeddingConfig::Voyage3_5_Lite_512 => {
galaxy_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35Lite512
}
EmbeddingConfig::Voyage4_512 => {
warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage4512
}
}
}
}
@@ -138,6 +144,9 @@ impl TryFrom<galaxy_graphql::full_source_code_embedding::EmbeddingConfig> for Em
galaxy_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35512 => {
Ok(Self::Voyage3_5_512)
}
warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage4512 => {
Ok(Self::Voyage4_512)
}
}
}
}
@@ -161,6 +170,36 @@ pub struct Fragment {
location: FragmentLocation,
}
impl Fragment {
pub fn from_byte_range(
content: String,
content_hash: ContentHash,
absolute_path: PathBuf,
byte_range: Range<ByteOffset>,
) -> Self {
Self {
content,
content_hash,
location: FragmentLocation {
absolute_path,
byte_range,
},
}
}
pub fn content_hash(&self) -> &ContentHash {
&self.content_hash
}
pub fn absolute_path(&self) -> &Path {
&self.location.absolute_path
}
pub fn byte_range(&self) -> Range<ByteOffset> {
self.location.byte_range.clone()
}
}
impl From<Fragment> for galaxy_graphql::full_source_code_embedding::Fragment {
fn from(val: Fragment) -> Self {
Self {
@@ -18,6 +18,14 @@ struct QueueEntry {
metadata: WorkspaceMetadata,
}
/// Controls whether queued builds may be consumed.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
enum BuildQueueState {
Paused,
#[default]
Running,
}
impl Hash for QueueEntry {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.metadata.path.hash(state);
@@ -35,6 +43,7 @@ impl Eq for QueueEntry {}
#[derive(Debug, Default)]
pub(super) struct BuildQueue {
queue: PriorityQueue<QueueEntry, Priority>,
state: BuildQueueState,
}
impl BuildQueue {
@@ -46,7 +55,10 @@ impl BuildQueue {
self.queue.iter().map(|(entry, _)| entry.metadata.clone())
}
pub(super) fn new_with_persisted(snapshots_to_load: Vec<WorkspaceMetadata>) -> Self {
pub(super) fn new_with_persisted(
snapshots_to_load: Vec<WorkspaceMetadata>,
start_immediately: bool,
) -> Self {
let mut queue = PriorityQueue::new();
queue.extend(
snapshots_to_load
@@ -54,12 +66,35 @@ impl BuildQueue {
.sorted_by(WorkspaceMetadata::most_recently_touched)
.map(|entry| (QueueEntry { metadata: entry }, Priority::PersistedSnapshot)),
);
let state = if start_immediately {
BuildQueueState::Running
} else {
BuildQueueState::Paused
};
Self { queue }
Self { queue, state }
}
pub(super) fn is_running(&self) -> bool {
self.state == BuildQueueState::Running
}
/// Starts consuming queued builds. Returns whether the queue transitioned to running.
pub(super) fn start(&mut self) -> bool {
match self.state {
BuildQueueState::Paused => {
self.state = BuildQueueState::Running;
true
}
BuildQueueState::Running => false,
}
}
/// Pulls the next index root path to sync from the priority queue and returns it.
pub fn pick_next_sync(&mut self) -> Option<WorkspaceMetadata> {
if !self.is_running() {
return None;
}
self.queue.pop().map(|(entry, _priority)| entry.metadata)
}
@@ -0,0 +1,178 @@
use std::collections::{HashMap, HashSet};
use std::ops::Range;
use std::path::PathBuf;
use super::{ContentHash, Fragment, FragmentLocation, FragmentMetadata};
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
#[derive(Default)]
pub struct ReadFragmentResult {
pub successfully_read: Vec<Fragment>,
pub fail_to_read: Vec<ContentHash>,
pub fail_to_read_path: Vec<PathBuf>,
}
pub fn build_fragments_from_file_contents(
metadatas: impl IntoIterator<Item = (ContentHash, FragmentMetadata)>,
file_contents: &HashMap<PathBuf, String>,
) -> ReadFragmentResult {
let mut fragments = Vec::new();
let mut fail_to_read = Vec::new();
let mut fail_to_read_path = Vec::new();
// Group fragments by file path.
let mut fragments_by_path: HashMap<_, Vec<_>> = HashMap::new();
for (content_hash, metadata) in metadatas {
fragments_by_path
.entry(metadata.absolute_path)
.or_default()
.push((content_hash, metadata.location.byte_range));
}
// Process each file and its fragments.
for (file_path, file_fragments) in fragments_by_path {
let mut has_failed_to_read_fragments = false;
if let Some(file_content) = file_contents.get(&file_path) {
// Process all fragments for this file.
for (content_hash, fragment_ranges) in file_fragments {
let start_idx = fragment_ranges.start.as_usize();
let end_idx = fragment_ranges.end.as_usize();
if start_idx <= end_idx
&& end_idx <= file_content.len()
&& file_content.is_char_boundary(start_idx)
&& file_content.is_char_boundary(end_idx)
{
let content = file_content[start_idx..end_idx].to_string();
if content.is_empty() {
log::trace!(
"Fragment for {:?} with range {:?} is empty",
file_path.display(),
fragment_ranges
);
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
} else if ContentHash::from_content(&content) != content_hash {
log::trace!(
"Fragment for {:?} with range {:?} does not match its content hash",
file_path.display(),
fragment_ranges
);
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
} else {
fragments.push(Fragment {
content,
content_hash,
location: FragmentLocation {
absolute_path: file_path.clone(),
byte_range: fragment_ranges,
},
});
}
} else {
log::trace!("Invalid byte range {fragment_ranges:?} for file: {file_path:?}");
fail_to_read.push(content_hash);
has_failed_to_read_fragments = true;
}
}
} else {
log::trace!("Failed to read file: {file_path:?}");
fail_to_read.extend(
file_fragments
.into_iter()
.map(|(content_hash, _)| content_hash),
);
has_failed_to_read_fragments = true;
}
if has_failed_to_read_fragments {
fail_to_read_path.push(file_path);
}
}
ReadFragmentResult {
successfully_read: fragments,
fail_to_read,
fail_to_read_path,
}
}
// Convert fragments into CodeContextLocations. This function groups and dedupes fragments in the same file.
// It also allows the caller to define a context line number surrounding the relevant fragment.
pub fn fragments_to_context_locations<'a>(
fragments: Vec<Fragment>,
metadata_for_hash: impl Fn(&ContentHash) -> Option<&'a [FragmentMetadata]>,
context_lines: usize,
) -> HashSet<CodeContextLocation> {
// Map to collect fragments by file path.
let mut fragments_by_path: HashMap<&PathBuf, Vec<Range<usize>>> = HashMap::new();
let mut whole_files = HashSet::new();
// First pass - collect all fragments and their line ranges by file path.
for fragment in &fragments {
if let Some(metadata) = metadata_for_hash(&fragment.content_hash).and_then(|metadatas| {
metadatas.iter().find(|m| {
m.absolute_path == fragment.location.absolute_path
&& m.location.byte_range == fragment.location.byte_range
})
}) {
// Add line range with context to the appropriate file's collection.
let path = &fragment.location.absolute_path;
let start = metadata.location.start_line.saturating_sub(context_lines);
let end = metadata.location.end_line + 1 + context_lines;
fragments_by_path.entry(path).or_default().push(start..end);
} else {
// Fallback to whole file if metadata not found.
whole_files.insert(fragment.location.absolute_path.clone());
}
}
// Second pass - process each file's fragments.
let mut result = HashSet::new();
// Process each file's fragments.
for (path, mut line_ranges) in fragments_by_path {
if line_ranges.is_empty() {
continue;
}
// We can skip the fragments if the entire file is already included in the context.
if whole_files.contains(path) {
continue;
}
// Sort ranges by start position.
line_ranges.sort_by_key(|range| range.start);
// Merge overlapping or adjacent ranges.
let mut merged_ranges: Vec<Range<usize>> = Vec::new();
for range in line_ranges {
if let Some(last) = merged_ranges.last_mut() {
// If current range overlaps or is adjacent to the last one, merge them.
if range.start <= last.end {
last.end = last.end.max(range.end);
} else {
merged_ranges.push(range);
}
} else {
merged_ranges.push(range);
}
}
// Add file fragment location with all merged ranges.
result.insert(CodeContextLocation::Fragment(FileFragmentLocation {
path: path.clone(),
line_ranges: merged_ranges,
}));
}
// Add whole files to the result set.
result.extend(whole_files.into_iter().map(CodeContextLocation::WholeFile));
result
}
#[cfg(test)]
#[path = "search_shaping_tests.rs"]
mod tests;
@@ -0,0 +1,133 @@
use std::collections::{HashMap, HashSet};
use std::ops::Range;
use std::path::PathBuf;
use string_offset::ByteOffset;
use super::super::{ContentHash, Fragment, FragmentLocation, FragmentMetadata};
use super::{build_fragments_from_file_contents, fragments_to_context_locations};
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
fn metadata(
path: &str,
byte_range: Range<ByteOffset>,
start_line: usize,
end_line: usize,
) -> FragmentMetadata {
FragmentMetadata {
absolute_path: PathBuf::from(path),
location: super::super::fragment_metadata::FragmentLocation {
start_line,
end_line,
byte_range,
},
}
}
fn fragment(content: &str, path: &str, byte_range: Range<ByteOffset>) -> Fragment {
Fragment {
content: content.to_string(),
content_hash: ContentHash::from_content(content),
location: FragmentLocation {
absolute_path: PathBuf::from(path),
byte_range,
},
}
}
#[test]
fn builds_fragments_from_exact_byte_ranges() {
let path = PathBuf::from("/repo/src/lib.rs");
let content = "before\nneedle\nπ-after".to_string();
let fragment_content = "needle";
let start = content.find(fragment_content).unwrap();
let end = start + fragment_content.len();
let content_hash = ContentHash::from_content(fragment_content);
let metadata = metadata(
path.to_string_lossy().as_ref(),
ByteOffset::from(start)..ByteOffset::from(end),
2,
2,
);
let result = build_fragments_from_file_contents(
[(content_hash.clone(), metadata)],
&HashMap::from([(path.clone(), content)]),
);
assert_eq!(result.fail_to_read.len(), 0);
assert_eq!(result.successfully_read.len(), 1);
let fragment = &result.successfully_read[0];
assert_eq!(fragment.content, fragment_content);
assert_eq!(fragment.content_hash, content_hash);
assert_eq!(fragment.location.absolute_path, path);
}
#[test]
fn rejects_invalid_hashes_and_byte_ranges() {
let path = PathBuf::from("/repo/src/lib.rs");
let content = "abcπdef".to_string();
let bad_hash_metadata = metadata(
path.to_string_lossy().as_ref(),
ByteOffset::from(0)..ByteOffset::from(3),
1,
1,
);
let invalid_boundary_metadata = metadata(
path.to_string_lossy().as_ref(),
ByteOffset::from(4)..ByteOffset::from(5),
1,
1,
);
let result = build_fragments_from_file_contents(
[
(ContentHash::from_content("not abc"), bad_hash_metadata),
(ContentHash::from_content("π"), invalid_boundary_metadata),
],
&HashMap::from([(path.clone(), content)]),
);
assert!(result.successfully_read.is_empty());
assert_eq!(result.fail_to_read.len(), 2);
assert_eq!(result.fail_to_read_path, vec![path]);
}
#[test]
fn shapes_fragments_into_merged_context_locations() {
let path = "/repo/src/lib.rs";
let fragment_a = fragment("a", path, ByteOffset::from(0)..ByteOffset::from(1));
let fragment_b = fragment("b", path, ByteOffset::from(2)..ByteOffset::from(3));
let metadata_a = metadata(path, ByteOffset::from(0)..ByteOffset::from(1), 10, 12);
let metadata_b = metadata(path, ByteOffset::from(2)..ByteOffset::from(3), 15, 17);
let metadata_by_hash = HashMap::from([
(fragment_a.content_hash.clone(), vec![metadata_a]),
(fragment_b.content_hash.clone(), vec![metadata_b]),
]);
let result = fragments_to_context_locations(
vec![fragment_a, fragment_b],
|hash| metadata_by_hash.get(hash).map(Vec::as_slice),
2,
);
assert_eq!(
result,
HashSet::from([CodeContextLocation::Fragment(FileFragmentLocation {
path: PathBuf::from(path),
line_ranges: std::iter::once(8..20).collect(),
})])
);
}
#[test]
fn falls_back_to_whole_file_when_metadata_is_missing() {
let path = "/repo/src/lib.rs";
let fragment = fragment("a", path, ByteOffset::from(0)..ByteOffset::from(1));
let result = fragments_to_context_locations(vec![fragment], |_| None, 2);
assert_eq!(
result,
HashSet::from([CodeContextLocation::WholeFile(PathBuf::from(path))])
);
}
@@ -1,20 +1,21 @@
use std::collections::HashSet;
use std::hash::{DefaultHasher, Hash, Hasher};
use std::path::{Path, PathBuf};
use std::time::Duration;
use chrono::Utc;
#[cfg(feature = "local_fs")]
use galaxyui::ModelHandle;
#[cfg(feature = "local_fs")]
use repo_metadata::Repository;
use std::{
collections::HashSet,
hash::{DefaultHasher, Hash, Hasher},
path::{Path, PathBuf},
time::Duration,
};
#[cfg(feature = "local_fs")]
use galaxyui_core::ModelHandle;
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
use super::Error as CodebaseIndexError;
use std::sync::Arc;
use galaxyui::ModelContext;
use galaxyui_core::ModelContext;
use anyhow::Context;
use galaxy_core::safe_info;
use super::{store_client::StoreClient, CodebaseIndex, EmbeddingConfig};
@@ -33,11 +34,47 @@ const REPO_SNAPSHOT_SHELF_LIFE_DURATION: Duration =
/// Subdirectory inside the app's statedirectory that holds snapshot files.
const REPO_SNAPSHOT_SUBDIR_NAME: &str = "codebase_index_snapshots";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SnapshotStorage {
dir: PathBuf,
}
impl SnapshotStorage {
/// Construct snapshot storage using the app's default secure snapshot directory.
pub fn app_default() -> Option<Self> {
snapshot_dir().map(|dir| Self { dir })
}
/// Construct snapshot storage rooted at the supplied directory, creating it if needed.
pub fn from_dir(dir: PathBuf) -> Option<Self> {
if !dir.is_dir() {
std::fs::create_dir_all(&dir).ok()?;
}
Some(Self { dir })
}
pub fn path(&self) -> &Path {
&self.dir
}
#[cfg(feature = "local_fs")]
pub(super) fn is_app_default(&self) -> bool {
self.dir == default_snapshot_dir_path()
}
pub(super) fn has_snapshot(&self, repo_path: &Path) -> bool {
self.snapshot_path(repo_path).is_file()
}
pub(super) fn snapshot_path(&self, repo_path: &Path) -> PathBuf {
snapshot_path(&self.dir, repo_path)
}
}
/// Splits a list of codebase indices into invalid and valid indices,
/// based on their last write date and whether they have a corresponding snapshot file.
pub(super) fn split_snapshot_metadata_by_validity(
persisted_codebase_indices: Vec<WorkspaceMetadata>,
snapshot_storage: Option<&SnapshotStorage>,
) -> (Vec<WorkspaceMetadata>, Vec<WorkspaceMetadata>) {
let now = Utc::now();
persisted_codebase_indices
@@ -48,7 +85,8 @@ pub(super) fn split_snapshot_metadata_by_validity(
index_metadata.path
);
index_metadata.is_expired(now, REPO_SNAPSHOT_SHELF_LIFE_DAYS)
|| !has_snapshot(&index_metadata.path)
|| !snapshot_storage
.is_some_and(|storage| storage.has_snapshot(&index_metadata.path))
})
}
@@ -159,12 +197,7 @@ pub(super) fn read_snapshot(
}
pub(super) fn has_snapshot(repo_path: &Path) -> bool {
let Some(snapshot_dir) = snapshot_dir() else {
return false;
};
let snapshot_path = snapshot_path(snapshot_dir.as_path(), repo_path);
snapshot_path.is_file()
SnapshotStorage::app_default().is_some_and(|storage| storage.has_snapshot(repo_path))
}
/// Construct a directory to store index snapshots, if it doesn't already exist,
@@ -175,9 +208,7 @@ pub(super) fn snapshot_dir() -> Option<PathBuf> {
#[cfg(feature = "local_fs")]
{
let base_dir =
galaxy_core::paths::secure_state_dir().unwrap_or_else(galaxy_core::paths::state_dir);
let snapshot_dir_path = base_dir.join(REPO_SNAPSHOT_SUBDIR_NAME);
let snapshot_dir_path = default_snapshot_dir_path();
if !snapshot_dir_path.is_dir() {
std::fs::create_dir_all(&snapshot_dir_path).ok()?;
@@ -186,9 +217,15 @@ pub(super) fn snapshot_dir() -> Option<PathBuf> {
}
}
#[cfg(feature = "local_fs")]
fn default_snapshot_dir_path() -> PathBuf {
galaxy_core::paths::secure_state_dir()
.unwrap_or_else(galaxy_core::paths::state_dir)
.join(REPO_SNAPSHOT_SUBDIR_NAME)
}
/// Constructs a snapshot path given a base directory and the codebase index's root path.
pub(super) fn snapshot_path(snapshot_dir: &Path, repo_path: &Path) -> PathBuf {
// Use a hash the repo_path to create a unique filename
// Use a hash of the repo_path to create a unique filename
let mut hasher = DefaultHasher::new();
repo_path.hash(&mut hasher);
let snapshot_file_name = format!("snapshot_{}", hasher.finish());
@@ -1,7 +1,57 @@
use chrono::Duration;
use chrono::{Duration, Utc};
use virtual_fs::{Stub, VirtualFS};
use super::*;
fn workspace_metadata(path: impl Into<PathBuf>) -> WorkspaceMetadata {
WorkspaceMetadata {
path: path.into(),
navigated_ts: None,
modified_ts: Some(Utc::now()),
queried_ts: None,
}
}
#[test]
#[cfg(feature = "local_fs")]
fn snapshot_storage_app_default_matches_snapshot_dir() {
VirtualFS::test(
"snapshot_storage_app_default_matches_snapshot_dir",
|_dirs, _sandbox| {
let storage = SnapshotStorage::app_default().unwrap();
assert_eq!(storage.path(), snapshot_dir().unwrap());
assert!(storage.is_app_default());
},
);
}
#[test]
fn split_snapshot_metadata_by_validity_uses_injected_snapshot_dir() {
let snapshot_dir = tempfile::tempdir().unwrap();
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
let repo_path = PathBuf::from("/remote/repo");
std::fs::write(storage.snapshot_path(&repo_path), b"snapshot").unwrap();
let (invalid_metadata, valid_metadata) =
split_snapshot_metadata_by_validity(vec![workspace_metadata(&repo_path)], Some(&storage));
assert!(invalid_metadata.is_empty());
assert_eq!(valid_metadata.len(), 1);
assert_eq!(valid_metadata[0].path, repo_path);
}
#[test]
fn split_snapshot_metadata_by_validity_rejects_missing_injected_snapshot() {
let snapshot_dir = tempfile::tempdir().unwrap();
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
let repo_path = PathBuf::from("/remote/repo");
let (invalid_metadata, valid_metadata) =
split_snapshot_metadata_by_validity(vec![workspace_metadata(&repo_path)], Some(&storage));
assert_eq!(invalid_metadata.len(), 1);
assert_eq!(invalid_metadata[0].path, repo_path);
assert!(valid_metadata.is_empty());
}
#[test]
fn test_clean_up_snapshot_files() {
@@ -71,12 +121,7 @@ fn test_clean_up_snapshot_files() {
.unwrap();
// Create test metadata that only includes the valid file
let metadata = vec![WorkspaceMetadata {
path: test_path,
navigated_ts: None,
modified_ts: None,
queried_ts: None,
}];
let metadata = vec![workspace_metadata(test_path)];
// Run cleanup
clean_up_snapshot_files(&snapshot_dir_absolute_path, &metadata);
@@ -1,9 +1,8 @@
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use std::time::Duration;
use async_trait::async_trait;
use std::{
collections::{HashMap, HashSet},
fmt::Debug,
time::Duration,
};
use super::{
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
@@ -1,28 +1,23 @@
use anyhow::{anyhow, Result};
use galaxy_core::sync_queue::{IsTransientError, SyncQueue, SyncQueueTaskTrait};
use itertools::Itertools;
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::mem;
use std::ops::AddAssign;
use std::pin::Pin;
use std::{
collections::{HashMap, HashSet},
mem,
sync::Arc,
};
use std::sync::Arc;
use super::{CodebaseContextConfig, NodeHash};
use crate::index::full_source_code_embedding::store_client::IntermediateNode;
use anyhow::{anyhow, Result};
use itertools::Itertools;
use galaxy_core::sync_queue::{IsTransientError, SyncQueue, SyncQueueTaskTrait};
use super::changed_files::ChangedFiles;
use super::codebase_index::{build_fragments_from_metadata, SyncProgress};
use super::fragment_metadata::LeafToFragmentMetadataMapping;
use super::merkle_tree::{MerkleTree, NodeLens};
use super::store_client::StoreClient;
use super::{
changed_files::ChangedFiles,
codebase_index::{build_fragments_from_metadata, SyncProgress},
fragment_metadata::LeafToFragmentMetadataMapping,
merkle_tree::{MerkleTree, NodeLens},
store_client::StoreClient,
EmbeddingConfig, Error, RepoMetadata,
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
};
use super::{ContentHash, Fragment};
use crate::index::full_source_code_embedding::store_client::IntermediateNode;
const SYNC_NODE_BATCH_SIZE: usize = 500;
// Minimum node batch size used for updates.
@@ -1,12 +1,11 @@
use std::collections::HashMap;
use futures::executor::block_on;
use virtual_fs::VirtualFS;
use super::batch_leaves_by_size;
use crate::index::full_source_code_embedding::merkle_tree::{construct_test_merkle_tree, NodeLens};
use virtual_fs::VirtualFS;
/// Collect all leaf nodes from a merkle tree by walking it recursively.
fn collect_leaves<'a>(node: NodeLens<'a>) -> Vec<NodeLens<'a>> {
if node.is_leaf() {
+3 -1
View File
@@ -1,5 +1,7 @@
use std::ops::Range;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use std::{ops::Range, path::PathBuf};
/// A line-based file fragment location.
///
+2 -4
View File
@@ -7,15 +7,13 @@ pub mod full_source_code_embedding;
#[cfg(feature = "local_fs")]
pub use file_outline::build_outline;
pub use file_outline::{Outline, Symbol};
pub use repo_metadata::{BuildTreeError, DirectoryEntry, Entry, FileId, FileMetadata};
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
pub use repo_metadata::{
matches_gitignores, path_passes_filters,
};
pub use repo_metadata::entry::{is_git_internal_path, should_watch_directory_in_git_path};
pub use repo_metadata::matches_gitignores;
}
}
+3
View File
@@ -1,6 +1,9 @@
pub mod agent;
pub mod api_keys;
pub mod aws_credentials;
pub mod geap_credentials;
#[cfg(not(target_family = "wasm"))]
pub mod grok_subscription;
pub mod llm_id;
pub use llm_id::LLMId;
+1 -2
View File
@@ -2,8 +2,7 @@ use galaxy_terminal::shell::ShellLaunchData;
use galaxy_util::path::{
convert_msys2_to_windows_native_path, convert_wsl_to_windows_host_path, msys2_exe_to_root,
};
use galaxyui::platform::OperatingSystem;
use typed_path::{TypedPath, TypedPathBuf, WindowsPath};
use galaxyui_core::platform::OperatingSystem;
fn use_unix_paths(shell: Option<&ShellLaunchData>) -> bool {
OperatingSystem::get().is_linux()
@@ -0,0 +1,21 @@
use warp_util::local_or_remote_path::LocalOrRemotePath;
use galaxyui_core::ModelContext;
use super::model::{ProjectContextModel, ProjectRule};
/// No-op stand-in for non-`local_fs` builds. File-based global rules require
/// filesystem watchers that don't exist on WASM, so callers see an empty
/// view here.
#[derive(Debug, Default)]
pub(crate) struct GlobalRules;
impl GlobalRules {
pub(crate) fn index(&mut self, _ctx: &mut ModelContext<ProjectContextModel>) {}
pub(crate) fn active_rules(&self) -> impl Iterator<Item = ProjectRule> + '_ {
std::iter::empty()
}
pub(crate) fn paths(&self) -> impl Iterator<Item = LocalOrRemotePath> + '_ {
std::iter::empty()
}
}
@@ -0,0 +1,394 @@
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use async_channel::Sender;
use repo_metadata::repository::{RepositorySubscriber, SubscriberId};
use repo_metadata::{DirectoryWatcher, Repository, RepositoryUpdate};
use strum::IntoEnumIterator;
use strum_macros::EnumIter;
use galaxy_core::safe_warn;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use warp_util::standardized_path::StandardizedPath;
use galaxyui_core::{ModelContext, ModelHandle, SingletonEntity};
use watcher::{HomeDirectoryWatcher, HomeDirectoryWatcherEvent};
use super::model::{GlobalRulesDelta, ProjectContextModel, ProjectContextModelEvent, ProjectRule};
/// A well-known location under `$HOME` that may contain a global rule file.
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)]
enum GlobalRuleSource {
/// `~/.agents/AGENTS.md`.
Agents,
}
impl GlobalRuleSource {
/// Display name (used in safe logs that don't expose user paths).
fn name(self) -> &'static str {
match self {
Self::Agents => "agents",
}
}
/// Subdirectory under `$HOME`, e.g. `".agents"`.
fn home_subdir(self) -> &'static str {
match self {
Self::Agents => ".agents",
}
}
/// File name within the subdir, e.g. `"AGENTS.md"`.
fn file_pattern(self) -> &'static str {
match self {
Self::Agents => "AGENTS.md",
}
}
}
#[derive(Debug)]
struct GlobalSourceWatcherState {
repository: ModelHandle<Repository>,
subscriber_id: SubscriberId,
}
#[derive(Debug)]
struct GlobalRulesUpdate {
/// The [`GlobalRuleSource`] variant that produced this update. The
/// receiver uses it to look up the matching `home_subdir`/`file_pattern`
/// without needing a per-source channel.
source: GlobalRuleSource,
update: RepositoryUpdate,
}
#[derive(Debug, Default)]
pub(crate) struct GlobalRules {
/// Global rule files keyed by absolute file path. Populated from
/// [`GlobalRuleSource`]. Independent of project-level rule indexing.
/// Stored in a `BTreeMap` so iteration order is deterministic.
pub(super) rules: BTreeMap<PathBuf, ProjectRule>,
/// Active home-subdir directory watchers, keyed by the absolute subdir
/// path (e.g. `~/.agents`).
source_watchers: HashMap<PathBuf, GlobalSourceWatcherState>,
/// Sender used by global-rule directory subscribers to push updates back
/// into the model's main-thread stream handler.
updates_tx: Option<Sender<GlobalRulesUpdate>>,
}
impl GlobalRules {
pub(crate) fn active_rules(&self) -> impl Iterator<Item = ProjectRule> + '_ {
self.rules.values().cloned()
}
pub(crate) fn paths(&self) -> impl Iterator<Item = LocalOrRemotePath> + '_ {
self.rules.keys().cloned().map(LocalOrRemotePath::Local)
}
/// Index all configured global rule sources (see [`GlobalRuleSource`]).
///
/// All disk I/O is dispatched through `ctx.spawn` so this method does not
/// block startup. Subscribes to [`HomeDirectoryWatcher`] to react to
/// creation/deletion of the home subdirs at runtime, and registers a
/// [`DirectoryWatcher`] per existing subdir for incremental updates.
///
/// Idempotent: subsequent calls are a no-op once the channel is initialized.
pub(crate) fn index(&mut self, ctx: &mut ModelContext<ProjectContextModel>) {
if self.updates_tx.is_some() {
return;
}
let Some(home_dir) = dirs::home_dir() else {
log::debug!("Home directory not found; skipping global rules indexing");
return;
};
// Set up the channel that all per-source subscribers push into.
let (tx, rx) = async_channel::unbounded::<GlobalRulesUpdate>();
self.updates_tx = Some(tx);
ctx.spawn_stream_local(
rx,
|me, update, ctx| {
me.global_rules
.handle_global_rules_update(update.source, update.update, ctx);
},
|_, _| {},
);
// React to creation/deletion of home subdirs at runtime.
ctx.subscribe_to_model(&HomeDirectoryWatcher::handle(ctx), |me, _, event, ctx| {
me.global_rules
.handle_home_dir_event_for_global_rules(event, ctx);
});
for source in GlobalRuleSource::iter() {
let subdir_path = home_dir.join(source.home_subdir());
let target_file = subdir_path.join(source.file_pattern());
// Initial async read; if the file doesn't exist yet, the watcher
// will pick it up on creation.
Self::spawn_global_rule_read(target_file, ctx);
if subdir_path.exists() {
self.register_global_source_watcher(source, &subdir_path, ctx);
}
}
}
/// Async read of a single global rule file. The async block runs on a
/// background executor; the main-thread callback updates model state once
/// the read completes.
fn spawn_global_rule_read(file_path: PathBuf, ctx: &mut ModelContext<ProjectContextModel>) {
ctx.spawn(
async move {
// `read_to_string` returning `Err` (e.g. NotFound, permission
// denied, file replaced with a non-regular file) is converted
// to `None`; the callback below decides whether that means
// "insert/refresh" or "drop a previously-known entry."
let content = async_fs::read_to_string(&file_path).await.ok();
(file_path, content)
},
move |me, (file_path, content_opt), ctx| match content_opt {
Some(content) => {
// Read succeeded: insert (or replace) the rule and notify
// subscribers.
me.global_rules.rules.insert(
file_path.clone(),
ProjectRule {
// Global rule sources are watched under the local home directory.
path: LocalOrRemotePath::Local(file_path.clone()),
content,
},
);
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
GlobalRulesDelta {
discovered_rules: vec![file_path],
deleted_rules: vec![],
},
));
}
None => {
// Drop cached content if file is now unreadable; no-op if it never existed.
if me.global_rules.rules.remove(&file_path).is_some() {
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
GlobalRulesDelta {
discovered_rules: vec![],
deleted_rules: vec![file_path],
},
));
}
}
},
);
}
/// Register a `DirectoryWatcher` on the given home subdir for incremental
/// updates. Idempotent: subsequent calls for an already-watched subdir are
/// a no-op (the `subdir_path` key dedups by directory rather than by
/// source, so multiple sources sharing a `home_subdir` would only register
/// the watcher once — a future change can fan out to multiple file
/// patterns by extending the value).
///
/// The subdir must exist on disk before this is called.
/// `DirectoryWatcher::add_directory` rejects non-existent paths, and
/// runtime creation is handled by `handle_home_dir_event_for_global_rules`,
/// which calls back here once the subdir appears.
fn register_global_source_watcher(
&mut self,
source: GlobalRuleSource,
subdir_path: &Path,
ctx: &mut ModelContext<ProjectContextModel>,
) {
// If the subdir is already being watched, return early.
if self.source_watchers.contains_key(subdir_path) {
return;
}
let (Some(update_tx), Ok(std_path)) = (
self.updates_tx.clone(),
StandardizedPath::from_local_canonicalized(subdir_path),
) else {
return;
};
let repo_handle = match DirectoryWatcher::handle(ctx)
.update(ctx, |watcher, ctx| watcher.add_directory(std_path, ctx))
{
Ok(handle) => handle,
Err(err) => {
// `safe_warn!` because the path contains the user's home dir,
// which is PII; we only want the full path on dogfood builds.
// The error itself can also embed the canonicalized path
// (e.g. `RepoMetadataError::RepoNotFound(...)`), so we keep
// it out of the safe branch as well — only the source name
// is safe to send to Sentry.
safe_warn!(
safe: (
"Failed to register {} for global rules watching",
source.name()
),
full: (
"Failed to register {} for global rules watching: {err}",
subdir_path.display()
)
);
return;
}
};
let subscriber = Box::new(GlobalRulesRepositorySubscriber { source, update_tx });
let start = repo_handle.update(ctx, |repo, ctx| repo.start_watching(subscriber, ctx));
let subscriber_id = start.subscriber_id;
let subdir_path_owned = subdir_path.to_path_buf();
self.source_watchers.insert(
subdir_path_owned.clone(),
GlobalSourceWatcherState {
repository: repo_handle.clone(),
subscriber_id,
},
);
let cleanup_key = subdir_path_owned.clone();
let subdir_for_log = subdir_path_owned;
ctx.spawn(start.registration_future, move |me, res, ctx| {
if let Err(err) = res {
// Same PII shape as the registration error above: the path
// and the error can both contain the user's home dir, so
// both stay in the `full` branch only.
safe_warn!(
safe: (
"Failed to start watching {} for global rules",
source.name()
),
full: (
"Failed to start watching {} for global rules: {err}",
subdir_for_log.display()
)
);
// Remove the stored watcher since registration failed.
if let Some(state) = me.global_rules.source_watchers.remove(&cleanup_key) {
state.repository.update(ctx, |repo, ctx| {
repo.stop_watching(state.subscriber_id, ctx);
});
}
}
});
}
/// Handle an incremental update for the given global source.
fn handle_global_rules_update(
&mut self,
source: GlobalRuleSource,
update: RepositoryUpdate,
ctx: &mut ModelContext<ProjectContextModel>,
) {
if update.is_empty() {
return;
}
let Some(home_dir) = dirs::home_dir() else {
return;
};
let target_file = home_dir
.join(source.home_subdir())
.join(source.file_pattern());
let was_deleted = update.deleted.iter().any(|f| f.path == target_file)
|| update.moved.values().any(|f| f.path == target_file);
let was_added_or_modified = update.added_or_modified().any(|f| f.path == target_file)
|| update.moved.keys().any(|f| f.path == target_file);
// If the file was deleted, remove it from the cached content and emit a change event.
if was_deleted && self.rules.remove(&target_file).is_some() {
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
GlobalRulesDelta {
discovered_rules: vec![],
deleted_rules: vec![target_file.clone()],
},
));
}
// If the file was added or modified, spawn a read to update the cached content.
if was_added_or_modified {
Self::spawn_global_rule_read(target_file, ctx);
}
}
/// React to creation/deletion of the registered home subdirs at runtime.
fn handle_home_dir_event_for_global_rules(
&mut self,
event: &HomeDirectoryWatcherEvent,
ctx: &mut ModelContext<ProjectContextModel>,
) {
let HomeDirectoryWatcherEvent::HomeFilesChanged(fs_event) = event;
let Some(home_dir) = dirs::home_dir() else {
log::warn!("Home directory not found; skipping global rules home dir event");
return;
};
for source in GlobalRuleSource::iter() {
let subdir_path = home_dir.join(source.home_subdir());
let subdir_deleted = fs_event.deleted.contains(&subdir_path)
|| fs_event.moved.values().any(|v| v == &subdir_path);
if subdir_deleted {
if let Some(state) = self.source_watchers.remove(&subdir_path) {
state.repository.update(ctx, |repo, ctx| {
repo.stop_watching(state.subscriber_id, ctx);
});
}
let target_file = subdir_path.join(source.file_pattern());
if self.rules.remove(&target_file).is_some() {
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
GlobalRulesDelta {
discovered_rules: vec![],
deleted_rules: vec![target_file],
},
));
}
}
let subdir_added =
fs_event.added.contains(&subdir_path) || fs_event.moved.contains_key(&subdir_path);
if subdir_added {
let target_file = subdir_path.join(source.file_pattern());
// Kick off the read first, then register the watcher for subsequent edits.
Self::spawn_global_rule_read(target_file, ctx);
self.register_global_source_watcher(source, &subdir_path, ctx);
}
}
}
}
/// Subscriber for a single global rules home subdir (e.g. `~/.agents`).
/// Tags every update with the originating [`GlobalRuleSource`] variant so the
/// model can dispatch to the right entry without per-source channels.
struct GlobalRulesRepositorySubscriber {
source: GlobalRuleSource,
update_tx: Sender<GlobalRulesUpdate>,
}
impl RepositorySubscriber for GlobalRulesRepositorySubscriber {
fn on_scan(
&mut self,
_repository: &Repository,
_ctx: &mut ModelContext<Repository>,
) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
// Initial-state read is performed separately by `spawn_global_rule_read`,
// so the on_scan event is intentionally a no-op.
Box::pin(async {})
}
fn on_files_updated(
&mut self,
_repository: &Repository,
update: &repo_metadata::RepositoryUpdate,
_ctx: &mut ModelContext<Repository>,
) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
let tx = self.update_tx.clone();
let source = self.source;
let update = update.clone();
Box::pin(async move {
let _ = tx.send(GlobalRulesUpdate { source, update }).await;
})
}
}
+9
View File
@@ -1 +1,10 @@
cfg_if::cfg_if! {
if #[cfg(feature = "local_fs")] {
mod global_rules;
pub(crate) use global_rules::GlobalRules;
} else {
mod dummy_global_rules;
pub(crate) use dummy_global_rules::GlobalRules;
}
}
pub mod model;
File diff suppressed because it is too large Load Diff
+523 -53
View File
@@ -1,10 +1,41 @@
use super::*;
use std::path::PathBuf;
use warp_util::host_id::HostId;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use warp_util::remote_path::RemotePath;
use warp_util::standardized_path::StandardizedPath;
fn local_path(path: &str) -> LocalOrRemotePath {
LocalOrRemotePath::Local(PathBuf::from(path))
}
fn insert_remote_project_rule(
model: &mut ProjectContextModel,
host_id: &str,
project_root: &str,
rule_path: &str,
content: &str,
) {
let rules = model
.path_to_rules
.entry(remote_path(host_id, project_root))
.or_default();
rules.upsert_rule(&remote_path(host_id, rule_path), content.to_string());
}
fn remote_path(host_id: &str, path: &str) -> LocalOrRemotePath {
LocalOrRemotePath::Remote(RemotePath::new(
HostId::new(host_id.to_string()),
StandardizedPath::try_new(path).unwrap(),
))
}
use super::*;
#[test]
fn test_find_applicable_rules_empty_rules() {
let rules = ProjectRules { rules: vec![] };
let path = PathBuf::from("/a/b/c/file.rs");
let path = local_path("/a/b/c/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert!(result.is_empty());
@@ -14,10 +45,10 @@ fn test_find_applicable_rules_empty_rules() {
fn test_find_applicable_rules_no_matching_rules() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/x/y/WARP.md"), "content1".to_string());
rules.upsert_rule(Path::new("/z/AGENTS.md"), "content2".to_string());
rules.upsert_rule(&local_path("/x/y/WARP.md"), "content1".to_string());
rules.upsert_rule(&local_path("/z/AGENTS.md"), "content2".to_string());
let path = PathBuf::from("/a/b/c/file.rs");
let path = local_path("/a/b/c/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert!(result.is_empty());
@@ -27,52 +58,52 @@ fn test_find_applicable_rules_no_matching_rules() {
fn test_find_applicable_rules_single_matching_rule() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/WARP.md"), "content1".to_string());
rules.upsert_rule(Path::new("/x/AGENTS.md"), "content2".to_string());
rules.upsert_rule(&local_path("/a/WARP.md"), "content1".to_string());
rules.upsert_rule(&local_path("/x/AGENTS.md"), "content2".to_string());
let path = PathBuf::from("/a/b/c/file.rs");
let path = local_path("/a/b/c/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
assert_eq!(result[0].path, local_path("/a/WARP.md"));
}
#[test]
fn test_find_applicable_rules_includes_all_ancestor_rules() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/WARP.md"), "root_warp".to_string());
rules.upsert_rule(Path::new("/a/b/WARP.md"), "nested_warp".to_string());
rules.upsert_rule(Path::new("/a/b/c/WARP.md"), "deep_warp".to_string());
rules.upsert_rule(&local_path("/a/WARP.md"), "root_warp".to_string());
rules.upsert_rule(&local_path("/a/b/WARP.md"), "nested_warp".to_string());
rules.upsert_rule(&local_path("/a/b/c/WARP.md"), "deep_warp".to_string());
let path = PathBuf::from("/a/b/c/d/file.rs");
let path = local_path("/a/b/c/d/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 3);
// All should be WARP.md files (same priority), order is not specified by depth
// Just verify all expected rules are present
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
assert!(paths.contains(&PathBuf::from("/a/WARP.md")));
assert!(paths.contains(&PathBuf::from("/a/b/WARP.md")));
assert!(paths.contains(&PathBuf::from("/a/b/c/WARP.md")));
let paths: Vec<LocalOrRemotePath> = result.iter().map(|r| r.path.clone()).collect();
assert!(paths.contains(&local_path("/a/WARP.md")));
assert!(paths.contains(&local_path("/a/b/WARP.md")));
assert!(paths.contains(&local_path("/a/b/c/WARP.md")));
}
#[test]
fn test_find_applicable_rules_multiple_patterns() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "agents_content".to_string());
rules.upsert_rule(Path::new("/a/WARP.md"), "warp_content".to_string());
rules.upsert_rule(&local_path("/a/b/AGENTS.md"), "agents_content".to_string());
rules.upsert_rule(&local_path("/a/WARP.md"), "warp_content".to_string());
let path = PathBuf::from("/a/b/file.rs");
let path = local_path("/a/b/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 2);
assert_eq!(result[0].path, PathBuf::from("/a/b/AGENTS.md"));
assert_eq!(result[0].path, local_path("/a/b/AGENTS.md"));
assert_eq!(result[0].content, "agents_content");
assert_eq!(result[1].path, PathBuf::from("/a/WARP.md"));
assert_eq!(result[1].path, local_path("/a/WARP.md"));
assert_eq!(result[1].content, "warp_content");
}
@@ -80,13 +111,13 @@ fn test_find_applicable_rules_multiple_patterns() {
fn test_find_applicable_rules_exact_path_match() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/b/WARP.md"), "exact_match".to_string());
rules.upsert_rule(&local_path("/a/b/WARP.md"), "exact_match".to_string());
let path = PathBuf::from("/a/b/file.rs");
let path = local_path("/a/b/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, PathBuf::from("/a/b/WARP.md"));
assert_eq!(result[0].path, local_path("/a/b/WARP.md"));
assert_eq!(result[0].content, "exact_match");
}
@@ -94,14 +125,14 @@ fn test_find_applicable_rules_exact_path_match() {
fn test_find_applicable_rules_ignores_deeper_paths() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/WARP.md"), "applicable".to_string());
rules.upsert_rule(Path::new("/a/b/c/d/e/WARP.md"), "too_deep".to_string()); // Path doesn't contain /a/b
rules.upsert_rule(&local_path("/a/WARP.md"), "applicable".to_string());
rules.upsert_rule(&local_path("/a/b/c/d/e/WARP.md"), "too_deep".to_string()); // Path doesn't contain /a/b
let path = PathBuf::from("/a/b/file.rs");
let path = local_path("/a/b/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
assert_eq!(result[0].path, local_path("/a/WARP.md"));
assert_eq!(result[0].content, "applicable");
}
@@ -109,13 +140,13 @@ fn test_find_applicable_rules_ignores_deeper_paths() {
fn test_find_applicable_rules_handles_root_path() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/WARP.md"), "root_rule".to_string());
rules.upsert_rule(&local_path("/WARP.md"), "root_rule".to_string());
let path = PathBuf::from("/a/b/file.rs");
let path = local_path("/a/b/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, PathBuf::from("/WARP.md"));
assert_eq!(result[0].path, local_path("/WARP.md"));
assert_eq!(result[0].content, "root_rule");
}
@@ -129,36 +160,36 @@ fn test_find_applicable_rules_complex_scenario() {
// All ancestor rule files should be included.
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/WARP.md"), "a_warp".to_string());
rules.upsert_rule(Path::new("/a/AGENTS.md"), "a_agents".to_string());
rules.upsert_rule(Path::new("/a/b/WARP.md"), "ab_warp".to_string());
rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "ab_agents".to_string());
rules.upsert_rule(Path::new("/x/WARP.md"), "irrelevant".to_string()); // Should be ignored
rules.upsert_rule(&local_path("/a/WARP.md"), "a_warp".to_string());
rules.upsert_rule(&local_path("/a/AGENTS.md"), "a_agents".to_string());
rules.upsert_rule(&local_path("/a/b/WARP.md"), "ab_warp".to_string());
rules.upsert_rule(&local_path("/a/b/AGENTS.md"), "ab_agents".to_string());
rules.upsert_rule(&local_path("/x/WARP.md"), "irrelevant".to_string()); // Should be ignored
let path = PathBuf::from("/a/b/c/file.rs");
let path = local_path("/a/b/c/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 4);
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
assert!(paths.contains(&PathBuf::from("/a/WARP.md")));
assert!(paths.contains(&PathBuf::from("/a/AGENTS.md")));
assert!(paths.contains(&PathBuf::from("/a/b/WARP.md")));
assert!(paths.contains(&PathBuf::from("/a/b/AGENTS.md")));
// Expect only WARP.md files to be included as they have higher priority.
assert_eq!(result[0].path, local_path("/a/WARP.md"));
assert_eq!(result[0].content, "a_warp");
assert_eq!(result[1].path, local_path("/a/b/WARP.md"));
assert_eq!(result[1].content, "ab_warp");
}
#[test]
fn test_find_applicable_rules_handles_unknown_file_patterns() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("/a/WARP.md"), "known_pattern".to_string());
rules.upsert_rule(Path::new("/a/UNKNOWN.md"), "unknown_pattern".to_string());
let path = PathBuf::from("/a/file.rs");
rules.upsert_rule(&local_path("/a/WARP.md"), "known_pattern".to_string());
rules.upsert_rule(&local_path("/a/UNKNOWN.md"), "unknown_pattern".to_string());
let path = local_path("/a/file.rs");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 1);
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
assert_eq!(result[0].path, local_path("/a/WARP.md"));
assert_eq!(result[0].content, "known_pattern");
}
@@ -166,20 +197,459 @@ fn test_find_applicable_rules_handles_unknown_file_patterns() {
fn test_find_applicable_rules_with_relative_paths() {
let mut rules = ProjectRules::default();
rules.upsert_rule(Path::new("src/WARP.md"), "src_warp".to_string());
rules.upsert_rule(&local_path("src/WARP.md"), "src_warp".to_string());
rules.upsert_rule(
Path::new("src/components/WARP.md"),
&local_path("src/components/WARP.md"),
"components_warp".to_string(),
);
let path = PathBuf::from("src/components/Button.tsx");
let path = local_path("src/components/Button.tsx");
let result = rules.find_active_or_applicable_rules(&path).active_rules;
assert_eq!(result.len(), 2);
// Both are WARP.md files (same priority), order within same priority is not guaranteed
// Just verify both rules are present
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
assert!(paths.contains(&PathBuf::from("src/WARP.md")));
assert!(paths.contains(&PathBuf::from("src/components/WARP.md")));
let paths: Vec<LocalOrRemotePath> = result.iter().map(|r| r.path.clone()).collect();
assert!(paths.contains(&local_path("src/WARP.md")));
assert!(paths.contains(&local_path("src/components/WARP.md")));
}
fn make_rule_path(path: &str) -> ProjectRulePath {
ProjectRulePath {
path: PathBuf::from(path),
project_root: PathBuf::from("/project"),
}
}
#[test]
fn test_merge_independent_deltas() {
let mut delta = RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
};
delta.merge(RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/b/WARP.md")],
});
assert_eq!(delta.discovered_rules.len(), 1);
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/b/WARP.md")]);
}
#[test]
fn test_merge_add_then_delete_yields_delete() {
let mut delta = RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
};
delta.merge(RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
});
assert!(delta.discovered_rules.is_empty());
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/a/WARP.md")]);
}
#[test]
fn test_merge_delete_then_add_yields_add() {
let mut delta = RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
};
delta.merge(RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
});
assert_eq!(delta.discovered_rules.len(), 1);
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
assert!(delta.deleted_rules.is_empty());
}
#[test]
fn test_merge_add_delete_add_yields_add() {
let mut delta = RulesDelta::default();
delta.merge(RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
});
delta.merge(RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
});
delta.merge(RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
});
assert_eq!(delta.discovered_rules.len(), 1);
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
assert!(delta.deleted_rules.is_empty());
}
#[test]
fn test_merge_delete_add_delete_yields_delete() {
let mut delta = RulesDelta::default();
delta.merge(RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
});
delta.merge(RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
});
delta.merge(RulesDelta {
discovered_rules: vec![],
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
});
assert!(delta.discovered_rules.is_empty());
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/a/WARP.md")]);
}
#[test]
fn test_merge_rediscovery_keeps_latest() {
let mut delta = RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
};
// A second discovery of the same path (content update) should deduplicate.
delta.merge(RulesDelta {
discovered_rules: vec![make_rule_path("/a/WARP.md")],
deleted_rules: vec![],
});
assert_eq!(delta.discovered_rules.len(), 1);
assert!(delta.deleted_rules.is_empty());
}
#[test]
fn test_missing_rule_content_preserves_cached_content_while_path_is_standing() {
let rule_path = local_path("/unavailable/project/WARP.md");
let mut existing_rules = ProjectRules::default();
existing_rules.upsert_rule(&rule_path, "cached content".to_string());
let rules = ProjectContextModel::reconcile_project_rules(
vec![rule_path.clone()],
Vec::new(),
existing_rules,
);
let result = rules.find_active_or_applicable_rules(&local_path("/unavailable/project/main.rs"));
assert_eq!(result.active_rules.len(), 1);
assert_eq!(result.active_rules[0].path, rule_path);
assert_eq!(result.active_rules[0].content, "cached content");
}
#[test]
fn test_rule_missing_from_standing_results_is_removed_from_cached_content() {
let rule_path = local_path("/unavailable/project/WARP.md");
let mut existing_rules = ProjectRules::default();
existing_rules.upsert_rule(&rule_path, "cached content".to_string());
let rules =
ProjectContextModel::reconcile_project_rules(Vec::new(), Vec::new(), existing_rules);
assert!(rules.rule_paths().next().is_none());
}
#[test]
fn test_reconcile_project_rules_hydrates_local_and_remote_paths() {
let local_rule_path = local_path("/local/WARP.md");
let remote_rule_path = remote_path("host-a", "/remote/AGENTS.md");
let rules = ProjectContextModel::reconcile_project_rules(
vec![local_rule_path.clone(), remote_rule_path.clone()],
vec![
(local_rule_path.clone(), "local content".to_string()),
(remote_rule_path.clone(), "remote content".to_string()),
],
ProjectRules::default(),
);
let local_result = rules.find_active_or_applicable_rules(&local_path("/local/main.rs"));
assert_eq!(local_result.active_rules.len(), 1);
assert_eq!(local_result.active_rules[0].path, local_rule_path);
assert_eq!(local_result.active_rules[0].content, "local content");
let remote_result =
rules.find_active_or_applicable_rules(&remote_path("host-a", "/remote/main.rs"));
assert_eq!(remote_result.active_rules.len(), 1);
assert_eq!(remote_result.active_rules[0].path, remote_rule_path);
assert_eq!(remote_result.active_rules[0].content, "remote content");
}
#[cfg(feature = "local_fs")]
#[test]
fn test_remote_standing_results_preserve_host_qualified_rule_paths() {
let host = HostId::new("test-host".to_string());
let repo_id = RepositoryIdentifier::Remote(RemotePath::new(
host.clone(),
StandardizedPath::try_new("/repo").unwrap(),
));
let rule_path = StandardizedPath::try_new("/repo/nested/WARP.md").unwrap();
let contents = [
StandingQueryContent::file(rule_path.clone()),
StandingQueryContent::directory(StandardizedPath::try_new("/repo/nested").unwrap()),
];
assert_eq!(
standing_project_rule_paths(&repo_id, &contents),
vec![LocalOrRemotePath::Remote(RemotePath::new(host, rule_path))]
);
}
// Helper for global-rules tests: inserts a synthetic global rule directly into
// the model. Bypasses the watcher infrastructure (which requires the warpui
// runtime) so we can exercise `find_applicable_rules`'s layering logic.
fn insert_global_rule(model: &mut ProjectContextModel, path: &Path, content: &str) {
model.global_rules.rules.insert(
path.to_path_buf(),
ProjectRule {
path: LocalOrRemotePath::Local(path.to_path_buf()),
content: content.to_string(),
},
);
}
fn insert_project_rule(
model: &mut ProjectContextModel,
project_root: &Path,
rule_path: &Path,
content: &str,
) {
let rules = model
.path_to_rules
.entry(LocalOrRemotePath::Local(project_root.to_path_buf()))
.or_default();
rules.upsert_rule(
&LocalOrRemotePath::Local(rule_path.to_path_buf()),
content.to_string(),
);
}
#[test]
fn test_remote_project_rules_require_matching_host() {
let mut model = ProjectContextModel::default();
insert_remote_project_rule(
&mut model,
"host-a",
"/repo",
"/repo/WARP.md",
"remote_project_rule",
);
let same_host = model
.find_applicable_project_rules(&remote_path("host-a", "/repo/src/main.rs"))
.expect("same-host remote rule should apply");
assert_eq!(same_host.root_path, remote_path("host-a", "/repo"));
assert_eq!(same_host.active_rules.len(), 1);
assert_eq!(same_host.active_rules[0].content, "remote_project_rule");
let other_host =
model.find_applicable_project_rules(&remote_path("host-b", "/repo/src/main.rs"));
assert!(other_host.is_none());
}
#[test]
fn test_global_rule_alone_no_project_rules() {
let mut model = ProjectContextModel::default();
insert_global_rule(
&mut model,
Path::new("/home/u/.agents/AGENTS.md"),
"global_content",
);
let result = model
.find_applicable_rules(&local_path("/some/project/file.rs"))
.expect("global rule should produce a result");
assert_eq!(result.active_rules.len(), 1);
assert_eq!(
result.active_rules[0].path,
local_path("/home/u/.agents/AGENTS.md")
);
assert_eq!(result.active_rules[0].content, "global_content");
assert!(result.additional_rule_paths.is_empty());
}
#[test]
fn test_global_rule_layered_with_project_warp() {
let mut model = ProjectContextModel::default();
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
insert_project_rule(
&mut model,
Path::new("/repo"),
Path::new("/repo/WARP.md"),
"project_warp",
);
let result = model
.find_applicable_rules(&local_path("/repo/src/main.rs"))
.expect("layered rules should produce a result");
// Layered precedence: global first, then project rules.
assert_eq!(result.active_rules.len(), 2);
assert_eq!(result.active_rules[0].content, "global");
assert_eq!(result.active_rules[1].content, "project_warp");
assert_eq!(result.root_path, local_path("/repo"));
}
#[test]
fn test_in_dir_warp_shadows_agents_with_global() {
let mut model = ProjectContextModel::default();
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
// Both WARP.md and AGENTS.md in the same project directory: WARP.md should
// shadow AGENTS.md (existing in-directory behavior preserved).
insert_project_rule(
&mut model,
Path::new("/repo"),
Path::new("/repo/WARP.md"),
"project_warp",
);
insert_project_rule(
&mut model,
Path::new("/repo"),
Path::new("/repo/AGENTS.md"),
"project_agents",
);
let result = model
.find_applicable_rules(&local_path("/repo/src/main.rs"))
.expect("layered rules should produce a result");
// Expect: [global, project WARP.md]. project AGENTS.md is shadowed.
assert_eq!(result.active_rules.len(), 2);
assert_eq!(result.active_rules[0].content, "global");
assert_eq!(result.active_rules[1].content, "project_warp");
}
#[test]
fn test_no_rules_returns_none() {
let model = ProjectContextModel::default();
let result = model.find_applicable_rules(&local_path("/some/path/file.rs"));
assert!(result.is_none());
}
#[test]
fn test_global_rule_root_path_falls_back_to_parent() {
let mut model = ProjectContextModel::default();
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
let result = model
.find_applicable_rules(&local_path("/some/file.rs"))
.expect("global rule should produce a result");
// No project root indexed; root_path falls back to parent of the global rule.
assert_eq!(result.root_path, local_path("/home/u/.agents"));
}
#[test]
fn test_multiple_global_rules_all_contribute() {
let mut model = ProjectContextModel::default();
insert_global_rule(
&mut model,
Path::new("/home/u/.agents/AGENTS.md"),
"agents_global",
);
insert_global_rule(
&mut model,
Path::new("/home/u/.warp/WARP.md"),
"warp_global",
);
let result = model
.find_applicable_rules(&local_path("/repo/src/main.rs"))
.expect("globals should produce a result");
assert_eq!(result.active_rules.len(), 2);
let contents: Vec<&str> = result
.active_rules
.iter()
.map(|r| r.content.as_str())
.collect();
assert!(contents.contains(&"agents_global"));
assert!(contents.contains(&"warp_global"));
}
#[test]
fn test_remote_global_rules_only_layer_for_matching_remote_host() {
let mut model = ProjectContextModel::default();
insert_global_rule(
&mut model,
Path::new("/home/local/.agents/AGENTS.md"),
"local_global",
);
insert_remote_project_rule(
&mut model,
"host-a",
"/repo",
"/repo/WARP.md",
"remote_project",
);
let host_a = HostId::new("host-a".to_string());
model.set_remote_global_rules(
host_a.clone(),
vec![ProjectRule {
path: remote_path("host-a", "/home/remote/.agents/AGENTS.md"),
content: "remote_global".to_string(),
}],
);
model.set_remote_global_rules(
HostId::new("host-b".to_string()),
vec![ProjectRule {
path: remote_path("host-b", "/home/remote/.agents/AGENTS.md"),
content: "other_remote_global".to_string(),
}],
);
let matching = model
.find_applicable_rules(&remote_path("host-a", "/repo/src/main.rs"))
.unwrap();
assert_eq!(
matching
.active_rules
.iter()
.map(|rule| rule.content.as_str())
.collect::<Vec<_>>(),
["local_global", "remote_global", "remote_project"]
);
let other_host = model
.find_applicable_rules(&remote_path("host-b", "/repo/src/main.rs"))
.unwrap();
assert_eq!(
other_host
.active_rules
.iter()
.map(|rule| rule.content.as_str())
.collect::<Vec<_>>(),
["local_global", "other_remote_global"]
);
let local = model
.find_applicable_rules(&local_path("/repo/src/main.rs"))
.unwrap();
assert_eq!(local.active_rules.len(), 1);
assert_eq!(local.active_rules[0].content, "local_global");
assert_eq!(
model.global_rule_paths().collect::<Vec<_>>(),
[local_path("/home/local/.agents/AGENTS.md")]
);
model.set_remote_global_rules(host_a, Vec::new());
let replaced = model
.find_applicable_rules(&remote_path("host-a", "/repo/src/main.rs"))
.unwrap();
assert_eq!(
replaced
.active_rules
.iter()
.map(|rule| rule.content.as_str())
.collect::<Vec<_>>(),
["local_global", "remote_project"]
);
}
+136 -9
View File
@@ -1,12 +1,14 @@
use std::path::PathBuf;
use crate::{
agent::action_result::{AnyFileContent, FileContext},
skills::{ParsedSkill, SkillProvider, SkillScope},
};
use warp_multi_agent_api as api;
use thiserror::Error;
use warp_multi_agent_api as api;
use warp_util::host_id::HostId;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use warp_util::remote_path::RemotePath;
use warp_util::standardized_path::StandardizedPath;
use crate::agent::action_result::{AnyFileContent, FileContext};
use crate::skills::{ParsedSkill, SkillProvider, SkillReference, SkillScope};
#[derive(Error, Debug)]
pub enum SkillConversionError {
@@ -22,14 +24,126 @@ pub enum SkillConversionError {
ProviderInvalid,
#[error("Invalid content")]
ContentInvalid,
#[error("Skill path origin is unavailable")]
PathOriginUnavailable,
#[error("Invalid remote skill path")]
RemotePathInvalid,
}
/// Identifies how a string skill path from an API payload should be interpreted.
///
/// Live agent responses can be decoded from the active session's location. Restored payloads do
/// not carry enough session identity to safely reconstruct path-based skill locations, so callers
/// must use [`SkillPathOrigin::Unavailable`] rather than silently assuming the local filesystem.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum SkillPathOrigin {
Local,
Remote {
host_id: HostId,
},
/// Path identity could not be restored, but the API payload already carries the skill
/// descriptor and content needed to render a historical transcript.
///
/// This intentionally uses a local path wrapper only as a display-compatible identity for
/// restored conversation UI. Live execution paths should use [`SkillPathOrigin::Local`] or
/// [`SkillPathOrigin::Remote`] so local/remote provenance is preserved.
RestoredDisplayOnly,
Unavailable,
}
impl SkillPathOrigin {
pub fn location_for_path(
&self,
path: impl Into<String>,
) -> Result<LocalOrRemotePath, SkillConversionError> {
let path = path.into();
match self {
SkillPathOrigin::Local | SkillPathOrigin::RestoredDisplayOnly => {
// Normalize the path to collapse duplicate separators (e.g. `//workspace/...`
// → `/workspace/...`) so skill cache lookups match the filesystem-derived keys.
// We operate on the raw string rather than using `PathBuf::components().collect()`
// because the latter re-serialises with platform-specific separators (backslashes
// on Windows) and treats leading `//` as a UNC prefix on Windows.
let normalized = collapse_slashes(&path);
Ok(LocalOrRemotePath::Local(PathBuf::from(normalized)))
}
SkillPathOrigin::Remote { host_id } => {
let path = StandardizedPath::try_new(&path)
.map_err(|_| SkillConversionError::RemotePathInvalid)?;
Ok(LocalOrRemotePath::Remote(RemotePath::new(
host_id.clone(),
path,
)))
}
SkillPathOrigin::Unavailable => Err(SkillConversionError::PathOriginUnavailable),
}
}
}
/// Collapse consecutive `/` separators into a single one.
///
/// Skill paths are always forward-slash POSIX-style paths on all platforms, so we normalise
/// at the string level rather than using [`std::path::PathBuf::components`], which would
/// re-serialise with backslashes on Windows and misinterpret `//prefix` as a UNC path.
fn collapse_slashes(path: &str) -> String {
let mut result = String::with_capacity(path.len());
let mut prev_slash = false;
for ch in path.chars() {
if ch == '/' {
if !prev_slash {
result.push(ch);
}
prev_slash = true;
} else {
result.push(ch);
prev_slash = false;
}
}
result
}
fn skill_reference_for_path(
path: impl Into<String>,
path_origin: &SkillPathOrigin,
) -> Result<SkillReference, SkillConversionError> {
path_origin
.location_for_path(path)
.map(SkillReference::Path)
}
pub fn skill_reference_from_api_skill_ref(
skill_ref: api::SkillRef,
path_origin: &SkillPathOrigin,
) -> Option<SkillReference> {
match skill_ref.skill_reference {
Some(api::skill_ref::SkillReference::Path(path)) => {
skill_reference_for_path(path, path_origin).ok()
}
Some(api::skill_ref::SkillReference::BundledSkillId(id)) => {
Some(SkillReference::BundledSkillId(id))
}
None => None,
}
}
pub fn skill_reference_from_read_skill_ref(
skill_reference: api::message::tool_call::read_skill::SkillReference,
path_origin: &SkillPathOrigin,
) -> Result<SkillReference, SkillConversionError> {
match skill_reference {
api::message::tool_call::read_skill::SkillReference::SkillPath(path) => {
skill_reference_for_path(path, path_origin)
}
api::message::tool_call::read_skill::SkillReference::BundledSkillId(id) => {
Ok(SkillReference::BundledSkillId(id))
}
}
}
impl From<ParsedSkill> for api::Skill {
fn from(skill: ParsedSkill) -> Self {
api::Skill {
descriptor: Some(api::SkillDescriptor {
skill_reference: Some(api::skill_descriptor::SkillReference::Path(
skill.path.to_string_lossy().to_string(),
skill.path.display_path(),
)),
name: skill.name,
description: skill.description,
@@ -37,7 +151,7 @@ impl From<ParsedSkill> for api::Skill {
provider: Some(skill.provider.into()),
}),
content: Some(api::FileContent {
file_path: skill.path.to_string_lossy().to_string(),
file_path: skill.path.display_path(),
content: skill.content,
line_range: skill
.line_range
@@ -89,6 +203,15 @@ impl TryFrom<api::Skill> for ParsedSkill {
type Error = SkillConversionError;
fn try_from(api_skill: api::Skill) -> Result<Self, Self::Error> {
Self::try_from_api_with_origin(api_skill, &SkillPathOrigin::Unavailable)
}
}
impl ParsedSkill {
pub fn try_from_api_with_origin(
api_skill: api::Skill,
path_origin: &SkillPathOrigin,
) -> Result<Self, SkillConversionError> {
let Some(descriptor) = api_skill.descriptor else {
return Err(SkillConversionError::MissingDescriptor);
};
@@ -121,7 +244,7 @@ impl TryFrom<api::Skill> for ParsedSkill {
let line_range = context.line_range.as_ref();
Ok(ParsedSkill {
path: PathBuf::from(&path),
path: path_origin.location_for_path(path)?,
name: descriptor.name,
description: descriptor.description,
content,
@@ -164,3 +287,7 @@ fn convert_provider(
api::skill_descriptor::provider::Type::OpenCode(_) => Ok(SkillProvider::OpenCode),
}
}
#[cfg(test)]
#[path = "conversion_tests.rs"]
mod conversion_tests;
+242
View File
@@ -0,0 +1,242 @@
use warp_multi_agent_api as api;
use warp_util::host_id::HostId;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use warp_util::remote_path::RemotePath;
use warp_util::standardized_path::StandardizedPath;
use super::{
skill_reference_from_api_skill_ref, skill_reference_from_read_skill_ref, SkillConversionError,
SkillPathOrigin,
};
use crate::skills::{ParsedSkill, SkillProvider, SkillReference, SkillScope};
fn api_project_skill(path: &str) -> api::Skill {
api::Skill {
descriptor: Some(api::SkillDescriptor {
skill_reference: Some(api::skill_descriptor::SkillReference::Path(
path.to_string(),
)),
name: "deploy".to_string(),
description: "Deploy the service".to_string(),
scope: Some(api::skill_descriptor::Scope {
r#type: Some(api::skill_descriptor::scope::Type::Project(())),
}),
provider: Some(api::skill_descriptor::Provider {
r#type: Some(api::skill_descriptor::provider::Type::Agents(())),
}),
}),
content: Some(api::FileContent {
file_path: path.to_string(),
content: "# Deploy".to_string(),
line_range: None,
}),
}
}
#[test]
fn try_from_api_with_remote_origin_preserves_host_identity() {
let host_id = HostId::new("remote-host".to_string());
let parsed = ParsedSkill::try_from_api_with_origin(
api_project_skill("/repo/.agents/skills/deploy/SKILL.md"),
&SkillPathOrigin::Remote {
host_id: host_id.clone(),
},
)
.expect("remote project skill should convert");
let LocalOrRemotePath::Remote(path) = parsed.path else {
panic!("expected a remote skill path");
};
assert_eq!(path.host_id, host_id);
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
}
#[test]
fn skill_ref_with_remote_origin_preserves_host_identity() {
let host_id = HostId::new("remote-host".to_string());
let skill_reference = skill_reference_from_api_skill_ref(
api::SkillRef {
skill_reference: Some(api::skill_ref::SkillReference::Path(
"/repo/.agents/skills/deploy/SKILL.md".to_string(),
)),
},
&SkillPathOrigin::Remote {
host_id: host_id.clone(),
},
);
let Some(SkillReference::Path(LocalOrRemotePath::Remote(path))) = skill_reference else {
panic!("expected a remote skill path");
};
assert_eq!(path.host_id, host_id);
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
}
#[test]
fn parsed_skill_api_conversion_emits_plain_path_reference() {
let skill_path = LocalOrRemotePath::Remote(RemotePath::new(
HostId::new("remote-host".to_string()),
StandardizedPath::try_new("/repo/.agents/skills/deploy/SKILL.md").unwrap(),
));
let api_skill: api::Skill = ParsedSkill {
path: skill_path.clone(),
name: "deploy".to_string(),
description: "Deploy the service".to_string(),
content: "# Deploy".to_string(),
line_range: None,
scope: SkillScope::Project,
provider: SkillProvider::Agents,
}
.into();
let descriptor = api_skill
.descriptor
.expect("converted skill should have descriptor");
assert_eq!(
descriptor.skill_reference,
Some(api::skill_descriptor::SkillReference::Path(
skill_path.display_path()
))
);
}
#[test]
fn skill_reference_api_conversion_emits_plain_path_reference() {
let skill_path = LocalOrRemotePath::Remote(RemotePath::new(
HostId::new("remote-host".to_string()),
StandardizedPath::try_new("/repo/.agents/skills/deploy/SKILL.md").unwrap(),
));
let reference: api::skill_descriptor::SkillReference =
SkillReference::Path(skill_path.clone()).into();
assert_eq!(
reference,
api::skill_descriptor::SkillReference::Path(skill_path.display_path())
);
}
#[test]
fn try_from_api_with_unavailable_origin_rejects_path_based_skills() {
let error = ParsedSkill::try_from_api_with_origin(
api_project_skill("/repo/.agents/skills/deploy/SKILL.md"),
&SkillPathOrigin::Unavailable,
)
.expect_err("restored skills without host context should not fabricate local paths");
assert!(matches!(error, SkillConversionError::PathOriginUnavailable));
}
#[test]
fn skill_ref_with_unavailable_origin_preserves_bundled_skills() {
let skill_reference = skill_reference_from_api_skill_ref(
api::SkillRef {
skill_reference: Some(api::skill_ref::SkillReference::BundledSkillId(
"review-comments".to_string(),
)),
},
&SkillPathOrigin::Unavailable,
);
assert_eq!(
skill_reference,
Some(SkillReference::BundledSkillId(
"review-comments".to_string()
))
);
}
#[test]
fn local_origin_normalizes_double_leading_slash() {
let result = SkillPathOrigin::Local
.location_for_path("//workspace/common-skills/.agents/skills/deploy/SKILL.md")
.expect("double-slash path should be accepted");
let LocalOrRemotePath::Local(path) = result else {
panic!("expected a local path");
};
assert_eq!(
path.to_str().unwrap(),
"/workspace/common-skills/.agents/skills/deploy/SKILL.md"
);
}
#[test]
fn local_origin_normalizes_multiple_slashes() {
let result = SkillPathOrigin::Local
.location_for_path("///workspace///skills///SKILL.md")
.expect("multi-slash path should be accepted");
let LocalOrRemotePath::Local(path) = result else {
panic!("expected a local path");
};
assert_eq!(path.to_str().unwrap(), "/workspace/skills/SKILL.md");
}
#[test]
fn local_origin_preserves_normal_absolute_path() {
let result = SkillPathOrigin::Local
.location_for_path("/workspace/.agents/skills/deploy/SKILL.md")
.expect("normal path should be accepted");
let LocalOrRemotePath::Local(path) = result else {
panic!("expected a local path");
};
assert_eq!(
path.to_str().unwrap(),
"/workspace/.agents/skills/deploy/SKILL.md"
);
}
#[test]
fn restored_display_origin_normalizes_double_leading_slash() {
let result = SkillPathOrigin::RestoredDisplayOnly
.location_for_path("//repo/.agents/skills/deploy/SKILL.md")
.expect("double-slash path should be accepted");
let LocalOrRemotePath::Local(path) = result else {
panic!("expected a local path");
};
assert_eq!(
path.to_str().unwrap(),
"/repo/.agents/skills/deploy/SKILL.md"
);
}
#[test]
fn read_skill_ref_with_local_origin_normalizes_double_slash() {
let skill_reference = skill_reference_from_read_skill_ref(
api::message::tool_call::read_skill::SkillReference::SkillPath(
"//workspace/.agents/skills/deploy/SKILL.md".to_string(),
),
&SkillPathOrigin::Local,
)
.expect("double-slash read_skill path should convert");
let SkillReference::Path(LocalOrRemotePath::Local(path)) = skill_reference else {
panic!("expected a local skill path");
};
assert_eq!(
path.to_str().unwrap(),
"/workspace/.agents/skills/deploy/SKILL.md"
);
}
#[test]
fn read_skill_ref_with_remote_origin_preserves_host_identity() {
let host_id = HostId::new("remote-host".to_string());
let skill_reference = skill_reference_from_read_skill_ref(
api::message::tool_call::read_skill::SkillReference::SkillPath(
"/repo/.agents/skills/deploy/SKILL.md".to_string(),
),
&SkillPathOrigin::Remote {
host_id: host_id.clone(),
},
)
.expect("remote read_skill skill references should convert");
let SkillReference::Path(LocalOrRemotePath::Remote(path)) = skill_reference else {
panic!("expected a remote skill path");
};
assert_eq!(path.host_id, host_id);
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
}
+9 -4
View File
@@ -4,11 +4,16 @@ mod parser;
mod read_skills;
mod skill_provider;
mod skill_reference;
pub use parse_skill::{parse_bundled_skill, parse_skill, ParsedSkill};
pub use conversion::{
skill_reference_from_api_skill_ref, skill_reference_from_read_skill_ref, SkillConversionError,
SkillPathOrigin,
};
pub use parse_skill::{
parse_bundled_skill, parse_skill, parse_skill_content_at_location, ParsedSkill,
};
pub use read_skills::read_skills;
pub use skill_provider::{
get_provider_for_path, home_skills_path, provider_rank, SkillProvider, SkillProviderDefinition,
SkillScope, SKILL_PROVIDER_DEFINITIONS,
get_provider_for_path, home_skills_path, provider_parent_directory_for_skills_root,
provider_rank, SkillProvider, SkillProviderDefinition, SkillScope, SKILL_PROVIDER_DEFINITIONS,
};
pub use skill_reference::SkillReference;
+68 -49
View File
@@ -1,13 +1,16 @@
use std::fmt::Display;
use std::fs;
use std::ops::Range;
use std::path::Path;
use anyhow::Result;
use lazy_static::lazy_static;
use regex::Regex;
use std::fmt::Display;
use std::ops::Range;
use std::path::{Path, PathBuf};
use super::parser::parse_markdown_file;
use super::skill_provider::{get_provider_for_path, get_scope_for_path, SkillProvider, SkillScope};
use thiserror::Error;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use super::parser::parse_markdown_content;
use super::skill_provider::{get_provider_for_path, get_scope_for_path, SkillProvider, SkillScope};
const MAX_SKILL_DESCRIPTION_CHARS: usize = 512;
@@ -17,6 +20,50 @@ lazy_static! {
static ref INCOMPLETE_SENTENCE: Regex =
Regex::new(r"[^.!?]*$").expect("Incomplete sentence regex should be valid");
}
/// Parse skill markdown content that was fetched outside the local filesystem.
///
/// This is used for remote project skills, whose SKILL.md body arrives through
/// the remote file-read transport rather than `std::fs`.
pub fn parse_skill_content_at_location(
path: LocalOrRemotePath,
content: &str,
provider: SkillProvider,
scope: SkillScope,
) -> Result<ParsedSkill> {
let parsed = parse_markdown_content(content)?;
let name = match parsed
.front_matter
.get("name")
.map(|value| value.trim())
.filter(|value| !value.is_empty())
{
Some(name) => name.to_string(),
None => derive_skill_name_from_path(&path)?,
};
let description = match parsed
.front_matter
.get("description")
.map(|value| value.trim())
.filter(|value| !value.is_empty())
{
Some(description) => description.to_string(),
None => truncate_skill_description(
&derive_description_from_content(&parsed.content, parsed.line_range.as_ref())
.unwrap_or_default(),
),
};
Ok(ParsedSkill {
path,
name,
description,
content: parsed.content,
line_range: parsed.line_range,
provider,
scope,
})
}
#[derive(Error, Debug)]
pub enum ParseSkillError {
@@ -29,7 +76,7 @@ pub enum ParseSkillError {
/// Represents a parsed skill with validated fields
#[derive(Debug, Clone, PartialEq)]
pub struct ParsedSkill {
pub path: PathBuf,
pub path: LocalOrRemotePath,
pub name: String,
pub description: String,
/// The entire content of the file (including front matter)
@@ -52,7 +99,7 @@ impl ParsedSkill {
impl Display for ParsedSkill {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Skill: {}", self.path.display())
write!(f, "Skill: {}", self.path.display_path())
}
}
@@ -64,9 +111,10 @@ impl Display for ParsedSkill {
/// # Returns
/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
pub fn parse_skill(path: &Path) -> Result<ParsedSkill> {
let provider = get_provider_for_path(path).unwrap_or(SkillProvider::Agents);
let provider_path = LocalOrRemotePath::Local(path.to_path_buf());
let provider = get_provider_for_path(&provider_path).unwrap_or(SkillProvider::Agents);
let scope = get_scope_for_path(path);
parse_skill_internal(path, provider, scope)
parse_local_skill_internal(path, provider, scope)
}
/// Parse a bundled skill markdown file.
@@ -81,55 +129,26 @@ pub fn parse_skill(path: &Path) -> Result<ParsedSkill> {
/// # Returns
/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
pub fn parse_bundled_skill(path: &Path) -> Result<ParsedSkill> {
parse_skill_internal(path, SkillProvider::Warp, SkillScope::Bundled)
parse_local_skill_internal(path, SkillProvider::Warp, SkillScope::Bundled)
}
fn parse_skill_internal(
fn parse_local_skill_internal(
path: &Path,
provider: SkillProvider,
scope: SkillScope,
) -> Result<ParsedSkill> {
let parsed = parse_markdown_file(path)?;
let name = match parsed
.front_matter
.get("name")
.map(|value| value.trim())
.filter(|value| !value.is_empty())
{
Some(name) => name.to_string(),
None => derive_skill_name_from_path(path)?,
};
let description = match parsed
.front_matter
.get("description")
.map(|value| value.trim())
.filter(|value| !value.is_empty())
{
Some(description) => description.to_string(),
None => truncate_skill_description(
&derive_description_from_content(&parsed.content, parsed.line_range.as_ref())
.unwrap_or_default(),
),
};
Ok(ParsedSkill {
path: path.to_path_buf(),
name,
description,
content: parsed.content,
line_range: parsed.line_range,
let content = fs::read_to_string(path)?;
parse_skill_content_at_location(
LocalOrRemotePath::Local(path.to_path_buf()),
&content,
provider,
scope,
})
)
}
fn derive_skill_name_from_path(path: &Path) -> Result<String> {
fn derive_skill_name_from_path(path: &LocalOrRemotePath) -> Result<String> {
path.parent()
.and_then(|parent| parent.file_name())
.and_then(|name| name.to_str())
.map(|name| name.to_string())
.and_then(|parent| parent.file_name().map(str::to_owned))
.ok_or(ParseSkillError::CouldNotDeriveSkillNameFromPath.into())
}
@@ -202,5 +221,5 @@ fn truncate_skill_description(description: &str) -> String {
}
#[cfg(test)]
#[path = "parse_skill_test.rs"]
#[path = "parse_skill_tests.rs"]
mod parse_skill_test;
@@ -1,5 +1,7 @@
use std::path::PathBuf;
use tempfile::TempDir;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use super::*;
@@ -48,7 +50,7 @@ Show concrete examples of using this Skill.
// Total of 12 lines, so line_range is 5..13
assert_eq!(result.line_range, Some(5..13));
// Verify path is the full file path
assert_eq!(result.path, skill_file);
assert_eq!(result.path, LocalOrRemotePath::Local(skill_file));
}
#[test]
+4 -18
View File
@@ -1,10 +1,9 @@
use std::collections::HashMap;
use std::ops::Range;
use anyhow::{Context, Result};
use regex::Regex;
use serde_yaml::Value;
use std::collections::HashMap;
use std::fs;
use std::ops::Range;
use std::path::Path;
/// Represents a parsed markdown file with YAML front matter
#[derive(Debug)]
@@ -20,19 +19,6 @@ pub struct ParsedMarkdown {
pub line_range: Option<Range<usize>>,
}
/// Parse a markdown file with YAML front matter
///
/// # Arguments
/// * `path` - Path to the markdown file to parse
///
/// # Returns
/// * `Result<ParsedMarkdown>` - Parsed document with front matter and content
#[allow(dead_code)]
pub fn parse_markdown_file(path: &Path) -> Result<ParsedMarkdown> {
let content = fs::read_to_string(path)?;
parse_markdown_content(&content)
}
/// Parse markdown content with YAML front matter
#[allow(dead_code)]
pub(crate) fn parse_markdown_content(content: &str) -> Result<ParsedMarkdown> {
@@ -97,5 +83,5 @@ pub(crate) fn parse_markdown_content(content: &str) -> Result<ParsedMarkdown> {
}
#[cfg(test)]
#[path = "parser_test.rs"]
#[path = "parser_tests.rs"]
mod parser_test;
+1 -1
View File
@@ -46,5 +46,5 @@ pub fn read_skills(path: &Path) -> Vec<ParsedSkill> {
}
#[cfg(test)]
#[path = "read_skills_test.rs"]
#[path = "read_skills_tests.rs"]
mod read_skills_test;
@@ -1,6 +1,9 @@
use super::*;
use std::fs;
use tempfile::tempdir;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use super::*;
#[test]
fn test_read_skills_with_valid_skills() {
@@ -49,7 +52,7 @@ This is the second test skill.
let skill1 = skills.iter().find(|s| s.name == "test-skill-1").unwrap();
assert_eq!(
skill1.path,
skill1_dir.join("SKILL.md").to_string_lossy().to_string()
LocalOrRemotePath::Local(skill1_dir.join("SKILL.md"))
);
assert_eq!(skill1.description, "First test skill");
assert!(skill1.content.contains("# Test Skill 1"));
@@ -60,7 +63,7 @@ This is the second test skill.
let skill2 = skills.iter().find(|s| s.name == "test-skill-2").unwrap();
assert_eq!(
skill2.path,
skill2_dir.join("SKILL.md").to_string_lossy().to_string()
LocalOrRemotePath::Local(skill2_dir.join("SKILL.md"))
);
assert_eq!(skill2.description, "Second test skill");
assert!(skill2.content.contains("# Test Skill 2"));
+60 -47
View File
@@ -3,16 +3,16 @@
//! This module defines the supported skill providers (i.e. Agents, Claude, Codex, Warp) and their
//! associated skills directory paths. It provides utilities for looking up providers
//! from paths and vice versa.
use dirs::home_dir;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use dirs::home_dir;
use serde::{Deserialize, Serialize};
use strum_macros::{Display, EnumString, VariantNames};
use galaxy_core::ui::color::CLAUDE_ORANGE;
use galaxy_core::ui::icons::Icon;
use galaxy_core::ui::theme::Fill;
use strum_macros::{Display, EnumString, VariantNames};
use galaxy_util::local_or_remote_path::LocalOrRemotePath;
/// Represents a skill provider/origin (Agents, Claude, Codex, or Warp).
#[derive(
@@ -167,29 +167,65 @@ pub fn home_skills_path(provider: SkillProvider) -> Option<PathBuf> {
home_dir().map(|home_dir| home_dir.join(&definition.skills_path))
}
/// Returns the skill provider for a given path, if it matches a known skill provider directory.
/// For example:
/// get_provider_for_path(Path::new("/repo/.claude/skills/my-skill/SKILL.md")) returns Some(SkillProvider::Claude).
/// Handles both SKILL.md files and files nested within a skill directory.
pub fn get_provider_for_path(path: &Path) -> Option<SkillProvider> {
let path_components: Vec<_> = path.components().collect();
/// Returns the skill provider for a location, if it matches a known skill provider directory.
///
/// Local locations retain home-directory-aware matching. All other locations are
/// classified by provider-directory structure using their standardized path representation.
pub fn get_provider_for_path(path: &LocalOrRemotePath) -> Option<SkillProvider> {
path.to_local_path()
.and_then(get_home_provider_for_local_path)
.or_else(|| get_provider_for_structural_path(path))
}
for def in SKILL_PROVIDER_DEFINITIONS.iter() {
if home_skills_path(def.provider)
.into_iter()
.any(|home_skills_path| path.starts_with(home_skills_path))
{
return Some(def.provider);
fn get_home_provider_for_local_path(path: &Path) -> Option<SkillProvider> {
SKILL_PROVIDER_DEFINITIONS
.iter()
.find(|definition| {
home_skills_path(definition.provider)
.into_iter()
.any(|home_skills_path| path.starts_with(home_skills_path))
})
.map(|definition| definition.provider)
}
/// Returns the directory containing a provider's skills root when `skills_root` has a known
/// provider directory suffix, preserving the original local or remote location encoding.
///
/// For example, `/repo/.agents/skills` resolves to `/repo`, regardless of whether the location
/// is encoded with Unix or Windows path separators.
pub fn provider_parent_directory_for_skills_root(
skills_root: &LocalOrRemotePath,
) -> Option<LocalOrRemotePath> {
match_provider_skills_root(skills_root).map(|(_, parent_directory)| parent_directory)
}
fn get_provider_for_structural_path(path: &LocalOrRemotePath) -> Option<SkillProvider> {
let mut current = Some(path.clone());
while let Some(candidate) = current {
if let Some((provider, _)) = match_provider_skills_root(&candidate) {
return Some(provider);
}
current = candidate.parent();
}
None
}
// Retrieves path components for the skill provider directory (i.e., [".claude", "skills"])
let skill_components: Vec<_> = def.skills_path.components().collect();
// Checks if some consecutive components of the path match the skill provider directory
for window in path_components.windows(skill_components.len()) {
if window == skill_components.as_slice() {
return Some(def.provider);
fn match_provider_skills_root(
skills_root: &LocalOrRemotePath,
) -> Option<(SkillProvider, LocalOrRemotePath)> {
for definition in SKILL_PROVIDER_DEFINITIONS.iter() {
let mut parent_directory = skills_root.clone();
let mut matches_provider = true;
for component in definition.skills_path.components().rev() {
let expected_component = component.as_os_str().to_str()?;
if parent_directory.file_name() != Some(expected_component) {
matches_provider = false;
break;
}
parent_directory = parent_directory.parent()?;
}
if matches_provider {
return Some((definition.provider, parent_directory));
}
}
None
@@ -211,28 +247,5 @@ pub fn get_scope_for_path(path: &Path) -> SkillScope {
}
#[cfg(test)]
mod tests {
use super::{
get_provider_for_path, get_scope_for_path, home_skills_path, SkillProvider, SkillScope,
};
#[test]
fn warp_home_skills_path_uses_warp_home_path() {
assert_eq!(
home_skills_path(SkillProvider::Warp),
galaxy_core::paths::galaxy_home_skills_dir()
);
}
#[test]
fn warp_home_skill_path_is_home_warp_skill() {
let Some(galaxy_home_skills_dir) = galaxy_core::paths::galaxy_home_skills_dir() else {
eprintln!("Skipping test: home directory not available");
return;
};
let path = galaxy_home_skills_dir.join("my-skill").join("SKILL.md");
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Warp));
assert_eq!(get_scope_for_path(&path), SkillScope::Home);
}
}
#[path = "skill_provider_tests.rs"]
mod tests;
@@ -0,0 +1,83 @@
use warp_util::host_id::HostId;
use warp_util::local_or_remote_path::LocalOrRemotePath;
use warp_util::remote_path::RemotePath;
use warp_util::standardized_path::StandardizedPath;
use super::{
get_provider_for_path, get_scope_for_path, home_skills_path,
provider_parent_directory_for_skills_root, SkillProvider, SkillScope,
};
#[test]
fn warp_home_skills_path_uses_warp_home_path() {
assert_eq!(
home_skills_path(SkillProvider::Warp),
galaxy_core::paths::warp_home_skills_dir()
);
}
#[test]
fn warp_home_skill_path_is_home_warp_skill() {
let Some(warp_home_skills_dir) = galaxy_core::paths::warp_home_skills_dir() else {
eprintln!("Skipping test: home directory not available");
return;
};
let path = warp_home_skills_dir.join("my-skill").join("SKILL.md");
assert_eq!(
get_provider_for_path(&LocalOrRemotePath::Local(path.clone())),
Some(SkillProvider::Warp)
);
assert_eq!(get_scope_for_path(&path), SkillScope::Home);
}
#[test]
fn remote_provider_path_is_classified_by_structure() {
let path = LocalOrRemotePath::Remote(RemotePath::new(
HostId::new("remote-host".to_string()),
StandardizedPath::try_new("/repo/.claude/skills/my-skill/SKILL.md").unwrap(),
));
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Claude));
}
#[test]
fn local_project_provider_path_is_classified_by_structure() {
let path = LocalOrRemotePath::Local(
std::env::temp_dir()
.join("repo")
.join(".claude")
.join("skills")
.join("my-skill")
.join("SKILL.md"),
);
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Claude));
}
#[test]
fn foreign_encoded_remote_provider_path_is_classified_by_structure() {
let path = LocalOrRemotePath::Remote(RemotePath::new(
HostId::new("remote-host".to_string()),
StandardizedPath::try_new(r"C:\repo\.codex\skills\my-skill\SKILL.md").unwrap(),
));
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Codex));
}
#[test]
fn foreign_encoded_remote_skills_root_resolves_provider_parent_directory() {
let host_id = HostId::new("remote-host".to_string());
let skills_root = LocalOrRemotePath::Remote(RemotePath::new(
host_id.clone(),
StandardizedPath::try_new(r"C:\repo\.agents\skills").unwrap(),
));
assert_eq!(
provider_parent_directory_for_skills_root(&skills_root),
Some(LocalOrRemotePath::Remote(RemotePath::new(
host_id,
StandardizedPath::try_new(r"C:\repo").unwrap(),
)))
);
}
+6 -6
View File
@@ -1,11 +1,13 @@
use std::fmt;
use serde::{Deserialize, Serialize};
use std::{fmt, path::PathBuf};
use warp_util::local_or_remote_path::LocalOrRemotePath;
/// An unique reference to a skill.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub enum SkillReference {
/// A skill identified by the path to its SKILL.md file.
Path(PathBuf),
Path(LocalOrRemotePath),
/// A bundled skill distributed with Warp.
BundledSkillId(String),
}
@@ -13,7 +15,7 @@ pub enum SkillReference {
impl fmt::Display for SkillReference {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SkillReference::Path(path) => path.display().fmt(f),
SkillReference::Path(path) => path.display_path().fmt(f),
SkillReference::BundledSkillId(id) => write!(f, "@warp-skill:{id}"),
}
}
@@ -23,9 +25,7 @@ impl From<SkillReference> for warp_multi_agent_api::skill_descriptor::SkillRefer
fn from(reference: SkillReference) -> Self {
match reference {
SkillReference::Path(path) => {
warp_multi_agent_api::skill_descriptor::SkillReference::Path(
path.to_string_lossy().to_string(),
)
warp_multi_agent_api::skill_descriptor::SkillReference::Path(path.display_path())
}
SkillReference::BundledSkillId(id) => {
warp_multi_agent_api::skill_descriptor::SkillReference::BundledSkillId(id)
+3 -5
View File
@@ -1,13 +1,11 @@
use std::time::Duration;
use galaxy_core::{
features::FeatureFlag,
register_telemetry_event,
telemetry::{EnablementState, TelemetryEvent, TelemetryEventDesc},
};
use serde::Serialize;
use serde_json::{json, Value};
use strum_macros::{EnumDiscriminants, EnumIter};
use galaxy_core::features::FeatureFlag;
use galaxy_core::register_telemetry_event;
use galaxy_core::telemetry::{EnablementState, TelemetryEvent, TelemetryEventDesc};
#[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
#[derive(Clone, EnumDiscriminants)]
+2 -1
View File
@@ -1,6 +1,7 @@
use chrono::{DateTime, Days, Utc};
use std::path::PathBuf;
use chrono::{DateTime, Days, Utc};
/// Public-facing metadata persisted in SQLite
#[derive(Debug, Default, Clone)]
pub struct WorkspaceMetadata {