first pass of merging in warp (doesn't build)
This commit is contained in:
@@ -1,26 +1,21 @@
|
||||
use std::{path::PathBuf, time::Duration};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use galaxy_core::features::FeatureFlag;
|
||||
use itertools::Itertools as _;
|
||||
use uuid::Uuid;
|
||||
use warp_multi_agent_api as api;
|
||||
|
||||
use crate::{
|
||||
agent::{
|
||||
action::{
|
||||
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
|
||||
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, ReadSkillRequest,
|
||||
SearchCodebaseRequest, ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest,
|
||||
UseComputerRequest,
|
||||
},
|
||||
action_result::{AnyFileContent, FileContext},
|
||||
convert::ToolToAIAgentActionError,
|
||||
FileLocations,
|
||||
},
|
||||
diff_validation::{ParsedDiff, V4AHunk},
|
||||
document::AIDocumentId,
|
||||
skills::SkillReference,
|
||||
use crate::agent::action::{
|
||||
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
|
||||
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, SearchCodebaseRequest,
|
||||
ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest, UseComputerRequest,
|
||||
};
|
||||
use crate::agent::action_result::{AnyFileContent, FileContext};
|
||||
use crate::agent::convert::ToolToAIAgentActionError;
|
||||
use crate::agent::FileLocations;
|
||||
use crate::diff_validation::{ParsedDiff, V4AHunk};
|
||||
use crate::document::AIDocumentId;
|
||||
|
||||
impl From<api::message::tool_call::RunShellCommand> for AIAgentActionType {
|
||||
fn from(value: api::message::tool_call::RunShellCommand) -> Self {
|
||||
@@ -430,10 +425,11 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
|
||||
.actions
|
||||
.into_iter()
|
||||
.map(|action| {
|
||||
let target = convert_computer_use_target(action.target);
|
||||
let Some(action_type) = action.r#type else {
|
||||
return Err(ToolToAIAgentActionError::MissingComputerUseActionType);
|
||||
};
|
||||
match action_type {
|
||||
let action = match action_type {
|
||||
use_computer::action::Type::MouseMove(mouse_move) => {
|
||||
Ok(computer_use::Action::MouseMove {
|
||||
to: coordinates_to_vec(mouse_move.to.as_ref())?,
|
||||
@@ -481,7 +477,9 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
|
||||
let key = convert_key(key_up.key)?;
|
||||
Ok(computer_use::Action::KeyUp { key })
|
||||
}
|
||||
}
|
||||
}?;
|
||||
log_window_target_coord(&action, target);
|
||||
Ok(computer_use::TargetedAction { action, target })
|
||||
})
|
||||
.try_collect()?;
|
||||
let screenshot_params = value
|
||||
@@ -505,19 +503,6 @@ impl From<api::message::tool_call::RequestComputerUse> for AIAgentActionType {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<api::message::tool_call::ReadSkill> for AIAgentActionType {
|
||||
type Error = ToolToAIAgentActionError;
|
||||
|
||||
fn try_from(value: api::message::tool_call::ReadSkill) -> Result<Self, Self::Error> {
|
||||
match value.skill_reference {
|
||||
Some(reference) => Ok(AIAgentActionType::ReadSkill(ReadSkillRequest {
|
||||
skill: SkillReference::from(reference),
|
||||
})),
|
||||
None => Err(ToolToAIAgentActionError::MissingSkillReference),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
|
||||
fn from(value: api::message::tool_call::FetchConversation) -> Self {
|
||||
AIAgentActionType::FetchConversation {
|
||||
@@ -526,18 +511,6 @@ impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<api::message::tool_call::read_skill::SkillReference> for SkillReference {
|
||||
fn from(value: api::message::tool_call::read_skill::SkillReference) -> Self {
|
||||
use warp_multi_agent_api::message::tool_call::read_skill::SkillReference as ApiSkillReference;
|
||||
match value {
|
||||
ApiSkillReference::SkillPath(skill_path) => {
|
||||
SkillReference::Path(PathBuf::from(skill_path))
|
||||
}
|
||||
ApiSkillReference::BundledSkillId(id) => SkillReference::BundledSkillId(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts API ScreenshotParams to the internal computer_use type.
|
||||
fn convert_screenshot_params(
|
||||
params: api::message::tool_call::ScreenshotParams,
|
||||
@@ -556,6 +529,51 @@ fn convert_screenshot_params(
|
||||
max_long_edge_px: (params.max_long_edge_px > 0).then_some(params.max_long_edge_px as usize),
|
||||
max_total_px: (params.max_total_px > 0).then_some(params.max_total_px as usize),
|
||||
region,
|
||||
target: convert_computer_use_target(params.target),
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an optional API `ComputerUseTarget` into the internal computer_use target. An absent
|
||||
/// or `Screen` target maps to the legacy whole-screen behavior.
|
||||
fn convert_computer_use_target(
|
||||
target: Option<api::message::tool_call::ComputerUseTarget>,
|
||||
) -> computer_use::Target {
|
||||
use api::message::tool_call::computer_use_target::Target as ApiTarget;
|
||||
match target.and_then(|t| t.target) {
|
||||
// The proto window id is an opaque string; on macOS it is a CGWindowID, so parse it back
|
||||
// to a u32. An unparseable id is treated as no valid window target and falls back to the
|
||||
// legacy whole-screen behavior rather than panicking.
|
||||
Some(ApiTarget::Window(window)) => match window.window_id.parse::<u32>() {
|
||||
Ok(window_id) => computer_use::Target::Window {
|
||||
window_id,
|
||||
pid: window.pid,
|
||||
},
|
||||
Err(_) => computer_use::Target::Screen,
|
||||
},
|
||||
Some(ApiTarget::Screen(_)) | None => computer_use::Target::Screen,
|
||||
}
|
||||
}
|
||||
|
||||
/// Logs the raw server-provided coordinates for a window-targeted computer-use action, so the
|
||||
/// agent-driven coordinate conversion can be compared against where the click actually lands.
|
||||
/// Gated on COMPUTER_USE_DEBUG and routed through `log` so it surfaces in the app's log file.
|
||||
fn log_window_target_coord(action: &computer_use::Action, target: computer_use::Target) {
|
||||
if std::env::var_os("COMPUTER_USE_DEBUG").is_none() {
|
||||
return;
|
||||
}
|
||||
let computer_use::Target::Window { window_id, pid } = target else {
|
||||
return;
|
||||
};
|
||||
let coord = match action {
|
||||
computer_use::Action::MouseMove { to } => Some(("mouse_move", to.x(), to.y())),
|
||||
computer_use::Action::MouseDown { at, .. } => Some(("mouse_down", at.x(), at.y())),
|
||||
computer_use::Action::MouseWheel { at, .. } => Some(("mouse_wheel", at.x(), at.y())),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((kind, x, y)) = coord {
|
||||
log::info!(
|
||||
"[computer_use] server->client {kind} window#={window_id} pid={pid} raw_coord=({x},{y})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,34 +1,32 @@
|
||||
mod convert;
|
||||
|
||||
use std::{fmt::Display, ops::Range, path::PathBuf, time::Duration};
|
||||
use std::fmt::Display;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use galaxy_terminal::model::BlockId;
|
||||
use itertools::Itertools as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use strum_macros::EnumDiscriminants;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
agent::{
|
||||
action_result::{
|
||||
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult,
|
||||
CreateDocumentsResult, EditDocumentsResult, FetchConversationResult, FileGlobResult,
|
||||
FileGlobV2Result, GrepResult, InsertReviewCommentsResult, ReadDocumentsResult,
|
||||
ReadFilesResult, ReadMCPResourceResult, ReadShellCommandOutputResult, ReadSkillResult,
|
||||
RequestCommandOutputResult, RequestComputerUseResult, RequestFileEditsResult,
|
||||
SearchCodebaseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
|
||||
SuggestNewConversationResult, SuggestPromptResult,
|
||||
TransferShellCommandControlToUserResult, UploadArtifactResult, UseComputerResult,
|
||||
WriteToLongRunningShellCommandResult,
|
||||
},
|
||||
AIAgentCitation, FileLocations,
|
||||
},
|
||||
diff_validation::ParsedDiff,
|
||||
document::AIDocumentId,
|
||||
skills::SkillReference,
|
||||
};
|
||||
pub use warp_multi_agent_api::LifecycleEventType;
|
||||
|
||||
use crate::agent::action_result::{
|
||||
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult, CreateDocumentsResult,
|
||||
EditDocumentsResult, FetchConversationResult, FileGlobResult, FileGlobV2Result, GrepResult,
|
||||
InsertReviewCommentsResult, ReadDocumentsResult, ReadFilesResult, ReadMCPResourceResult,
|
||||
ReadShellCommandOutputResult, ReadSkillResult, RequestCommandOutputResult,
|
||||
RequestComputerUseResult, RequestFileEditsResult, RunAgentsResult, SearchCodebaseResult,
|
||||
SendMessageToAgentResult, StartAgentResult, StartAgentVersion, SuggestNewConversationResult,
|
||||
SuggestPromptResult, TransferShellCommandControlToUserResult, UploadArtifactResult,
|
||||
UseComputerResult, WaitForEventsResult, WriteToLongRunningShellCommandResult,
|
||||
};
|
||||
use crate::agent::{AIAgentCitation, FileLocations};
|
||||
use crate::diff_validation::ParsedDiff;
|
||||
use crate::document::AIDocumentId;
|
||||
use crate::skills::SkillReference;
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq, EnumDiscriminants)]
|
||||
pub enum AIAgentActionType {
|
||||
/// The AI requested the output for a given command to be retrieved as context in responding to
|
||||
@@ -48,7 +46,7 @@ pub enum AIAgentActionType {
|
||||
/// result instead.
|
||||
wait_until_completion: bool,
|
||||
|
||||
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke pager.
|
||||
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke a pager.
|
||||
uses_pager: Option<bool>,
|
||||
|
||||
/// The AI's rationale for requesting a command.
|
||||
@@ -144,7 +142,9 @@ pub enum AIAgentActionType {
|
||||
FetchConversation {
|
||||
conversation_id: String,
|
||||
},
|
||||
|
||||
// TODO(QUALITY-788): Delete legacy start_agent/start_agent_v2 action support once
|
||||
// old preview orchestration history no longer needs parse/display/result compatibility.
|
||||
// Linear issue: QUALITY-788.
|
||||
StartAgent {
|
||||
version: StartAgentVersion,
|
||||
name: String,
|
||||
@@ -167,6 +167,68 @@ pub enum AIAgentActionType {
|
||||
AskUserQuestion {
|
||||
questions: Vec<AskUserQuestionItem>,
|
||||
},
|
||||
|
||||
/// AI requested batched orchestration of one-or-more child agents that
|
||||
/// share run-wide configuration (model, harness, execution mode).
|
||||
/// The full per-child prompt is computed at dispatch time as
|
||||
/// `base_prompt + "\n\n" + agent_run_configs[i].prompt` (or just
|
||||
/// `base_prompt` when the per-agent `prompt` is empty).
|
||||
RunAgents(RunAgentsRequest),
|
||||
|
||||
/// Synthesized from a server-emitted Message::ToolCall::WaitForEvents;
|
||||
/// dispatched by WaitForEventsExecutor.
|
||||
WaitForEvents {
|
||||
/// tool_call_id of the unresolved WaitForEvents call; used to
|
||||
/// match inbound resume signals.
|
||||
tool_call_id: String,
|
||||
/// 0 means "unset" (prost flat-scalar convention); the executor
|
||||
/// falls back to a default.
|
||||
idle_timeout_seconds: i32,
|
||||
},
|
||||
}
|
||||
|
||||
/// Run-wide + per-agent configuration for a `RunAgents` tool call.
|
||||
///
|
||||
/// Mirrors the proto `RunAgents` message. Server-resolved fields
|
||||
/// (`model_id`, `harness_type`, `execution_mode`'s remote details) are
|
||||
/// folded in by the server's final tool-call re-emission once the
|
||||
/// payload is complete; the client renders the full layout from a
|
||||
/// fully-resolved instance only.
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct RunAgentsRequest {
|
||||
pub summary: String,
|
||||
pub base_prompt: String,
|
||||
pub skills: Vec<SkillReference>,
|
||||
pub model_id: String,
|
||||
pub harness_type: String,
|
||||
pub execution_mode: RunAgentsExecutionMode,
|
||||
pub agent_run_configs: Vec<RunAgentsAgentRunConfig>,
|
||||
pub plan_id: String,
|
||||
/// Resolved client-side at dispatch time; not serialized to the wire.
|
||||
pub harness_auth_secret_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub enum RunAgentsExecutionMode {
|
||||
Local,
|
||||
Remote {
|
||||
environment_id: String,
|
||||
worker_host: String,
|
||||
computer_use_enabled: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl RunAgentsExecutionMode {
|
||||
pub fn is_remote(&self) -> bool {
|
||||
matches!(self, Self::Remote { .. })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct RunAgentsAgentRunConfig {
|
||||
pub name: String,
|
||||
pub prompt: String,
|
||||
pub title: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
@@ -175,6 +237,11 @@ pub enum StartAgentExecutionMode {
|
||||
/// `None` selects the legacy embedded local child-agent flow.
|
||||
/// `Some(...)` selects a third-party CLI harness to launch locally.
|
||||
harness_type: Option<String>,
|
||||
/// `None` inherits the parent agent's preferred LLM (legacy behavior).
|
||||
/// `Some(_)` overrides the child's preferred LLM with the supplied
|
||||
/// model id (used by the orchestrate confirmation card so the user's
|
||||
/// model selection is honored on local launches).
|
||||
model_id: Option<String>,
|
||||
},
|
||||
Remote {
|
||||
environment_id: String,
|
||||
@@ -184,18 +251,27 @@ pub enum StartAgentExecutionMode {
|
||||
worker_host: String,
|
||||
harness_type: String,
|
||||
title: String,
|
||||
/// Name of a managed secret to forward as the authentication
|
||||
/// credential for the remote child when running a non-Oz harness.
|
||||
/// `None` means no client-side secret was selected — the remote
|
||||
/// environment falls back to its own ambient credentials.
|
||||
auth_secret_name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl StartAgentExecutionMode {
|
||||
/// Constructs a local execution mode using the legacy v1 default harness.
|
||||
pub fn local_with_defaults() -> Self {
|
||||
Self::Local { harness_type: None }
|
||||
Self::Local {
|
||||
harness_type: None,
|
||||
model_id: None,
|
||||
}
|
||||
}
|
||||
/// Constructs a local execution mode for a specific third-party harness.
|
||||
pub fn local_harness(harness_type: String) -> Self {
|
||||
Self::Local {
|
||||
harness_type: Some(harness_type),
|
||||
model_id: None,
|
||||
}
|
||||
}
|
||||
/// Constructs a remote execution mode using the legacy v1 defaults for
|
||||
@@ -209,6 +285,7 @@ impl StartAgentExecutionMode {
|
||||
worker_host: String::new(),
|
||||
harness_type: String::new(),
|
||||
title: String::new(),
|
||||
auth_secret_name: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -317,6 +394,10 @@ impl AIAgentActionType {
|
||||
Self::AskUserQuestion { .. } => {
|
||||
AIAgentActionResultType::AskUserQuestion(AskUserQuestionResult::Cancelled)
|
||||
}
|
||||
Self::RunAgents(_) => AIAgentActionResultType::RunAgents(RunAgentsResult::Cancelled),
|
||||
Self::WaitForEvents { .. } => {
|
||||
AIAgentActionResultType::WaitForEvents(WaitForEventsResult::Cancelled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,6 +443,10 @@ impl AIAgentActionType {
|
||||
Self::AskUserQuestion { questions } => {
|
||||
format!("Ask user {} question(s)", questions.len())
|
||||
}
|
||||
Self::RunAgents(req) => {
|
||||
format!("Orchestrate {} agent(s)", req.agent_run_configs.len())
|
||||
}
|
||||
Self::WaitForEvents { .. } => "Wait for events".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -534,6 +619,24 @@ impl Display for AIAgentActionType {
|
||||
AIAgentActionType::AskUserQuestion { questions } => {
|
||||
write!(f, "AskUserQuestion: {} question(s)", questions.len())
|
||||
}
|
||||
AIAgentActionType::RunAgents(req) => {
|
||||
let names = req
|
||||
.agent_run_configs
|
||||
.iter()
|
||||
.map(|c| c.name.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
write!(f, "Orchestrate: summary='{}' agents=[{names}]", req.summary,)
|
||||
}
|
||||
AIAgentActionType::WaitForEvents {
|
||||
tool_call_id,
|
||||
idle_timeout_seconds,
|
||||
} => {
|
||||
write!(
|
||||
f,
|
||||
"WaitForEvents: tool_call_id={tool_call_id} idle_timeout_seconds={idle_timeout_seconds}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -665,7 +768,8 @@ pub struct CreateDocumentsRequest {
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct UseComputerRequest {
|
||||
pub action_summary: String,
|
||||
pub actions: Vec<computer_use::Action>,
|
||||
/// Each action carries the surface (screen or a specific window) it targets.
|
||||
pub actions: Vec<computer_use::TargetedAction>,
|
||||
/// If set, a screenshot will be captured after the actions are executed.
|
||||
pub screenshot_params: Option<computer_use::ScreenshotParams>,
|
||||
}
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
use warp_multi_agent_api::{
|
||||
self as api,
|
||||
apply_file_diffs_result::success::UpdatedFileContent,
|
||||
ask_user_question_result::answer_item::{self, Answer as AskUserQuestionAnswer},
|
||||
use chrono::{DateTime, Local};
|
||||
use warp_multi_agent_api::apply_file_diffs_result::success::UpdatedFileContent;
|
||||
use warp_multi_agent_api::ask_user_question_result::answer_item::{
|
||||
self, Answer as AskUserQuestionAnswer,
|
||||
};
|
||||
|
||||
use crate::agent::{action_result::ShellCommandError, convert::ConvertToAPITypeError};
|
||||
use warp_multi_agent_api::{self as api};
|
||||
|
||||
use super::*;
|
||||
use crate::agent::action_result::ShellCommandError;
|
||||
use crate::agent::convert::ConvertToAPITypeError;
|
||||
|
||||
fn local_datetime_to_timestamp(timestamp: DateTime<Local>) -> prost_types::Timestamp {
|
||||
prost_types::Timestamp {
|
||||
seconds: timestamp.timestamp(),
|
||||
nanos: timestamp.timestamp_subsec_nanos() as i32,
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_result::Result {
|
||||
type Error = ConvertToAPITypeError;
|
||||
@@ -18,7 +26,8 @@ impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_resu
|
||||
block_id,
|
||||
output,
|
||||
exit_code,
|
||||
..
|
||||
start_ts,
|
||||
completed_ts,
|
||||
} => Ok(
|
||||
api::request::input::tool_call_result::Result::RunShellCommand(
|
||||
#[allow(deprecated)]
|
||||
@@ -31,6 +40,8 @@ impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_resu
|
||||
command_id: block_id.to_string(),
|
||||
output,
|
||||
exit_code: exit_code.value(),
|
||||
start_ts: start_ts.map(local_datetime_to_timestamp),
|
||||
finish_ts: completed_ts.map(local_datetime_to_timestamp),
|
||||
},
|
||||
)),
|
||||
},
|
||||
@@ -127,7 +138,7 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
|
||||
},
|
||||
),
|
||||
),
|
||||
WriteToLongRunningShellCommandResult::CommandFinished { block_id, output, exit_code, .. } => Ok(
|
||||
WriteToLongRunningShellCommandResult::CommandFinished { block_id, output, exit_code, start_ts, completed_ts } => Ok(
|
||||
api::request::input::tool_call_result::Result::WriteToLongRunningShellCommand(
|
||||
api::WriteToLongRunningShellCommandResult {
|
||||
result: Some(api::write_to_long_running_shell_command_result::Result::CommandFinished(
|
||||
@@ -135,6 +146,8 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
|
||||
command_id: block_id.to_string(),
|
||||
output,
|
||||
exit_code: exit_code.value(),
|
||||
start_ts: start_ts.map(local_datetime_to_timestamp),
|
||||
finish_ts: completed_ts.map(local_datetime_to_timestamp),
|
||||
}
|
||||
))
|
||||
},
|
||||
@@ -797,7 +810,8 @@ impl TryFrom<ReadShellCommandOutputResult> for api::request::input::tool_call_re
|
||||
block_id,
|
||||
output,
|
||||
exit_code,
|
||||
..
|
||||
start_ts,
|
||||
completed_ts,
|
||||
} => Ok(
|
||||
api::request::input::tool_call_result::Result::ReadShellCommandOutput(
|
||||
api::ReadShellCommandOutputResult {
|
||||
@@ -807,6 +821,8 @@ impl TryFrom<ReadShellCommandOutputResult> for api::request::input::tool_call_re
|
||||
command_id: block_id.to_string(),
|
||||
output,
|
||||
exit_code: exit_code.value(),
|
||||
start_ts: start_ts.map(local_datetime_to_timestamp),
|
||||
finish_ts: completed_ts.map(local_datetime_to_timestamp),
|
||||
},
|
||||
)),
|
||||
},
|
||||
@@ -891,6 +907,8 @@ impl TryFrom<TransferShellCommandControlToUserResult>
|
||||
block_id,
|
||||
output,
|
||||
exit_code,
|
||||
start_ts,
|
||||
completed_ts,
|
||||
} => Ok(
|
||||
api::request::input::tool_call_result::Result::TransferShellCommandControlToUser(
|
||||
api::TransferShellCommandControlToUserResult {
|
||||
@@ -900,6 +918,8 @@ impl TryFrom<TransferShellCommandControlToUserResult>
|
||||
command_id: block_id.to_string(),
|
||||
output,
|
||||
exit_code: exit_code.value(),
|
||||
start_ts: start_ts.map(local_datetime_to_timestamp),
|
||||
finish_ts: completed_ts.map(local_datetime_to_timestamp),
|
||||
},
|
||||
),
|
||||
),
|
||||
@@ -1109,6 +1129,7 @@ impl TryFrom<RequestComputerUseResult> for api::request::input::tool_call_result
|
||||
RequestComputerUseResult::Approved {
|
||||
screenshot,
|
||||
platform,
|
||||
windows,
|
||||
} => Ok(
|
||||
api::request::input::tool_call_result::Result::RequestComputerUse(
|
||||
api::RequestComputerUseResult {
|
||||
@@ -1125,6 +1146,7 @@ impl TryFrom<RequestComputerUseResult> for api::request::input::tool_call_result
|
||||
height: screenshot.height as i32,
|
||||
}),
|
||||
platform: convert_platform(platform).into(),
|
||||
windows: windows.into_iter().map(convert_window_info).collect(),
|
||||
},
|
||||
)),
|
||||
},
|
||||
@@ -1158,6 +1180,9 @@ impl TryFrom<UseComputerResult> for api::request::input::tool_call_result::Resul
|
||||
fn try_from(result: UseComputerResult) -> Result<Self, Self::Error> {
|
||||
match result {
|
||||
UseComputerResult::Success(result) => {
|
||||
// Copy out the captured-window metadata (if any) before the owned fields of
|
||||
// `result` are moved into the message below.
|
||||
let captured = result.captured_window;
|
||||
Ok(api::request::input::tool_call_result::Result::UseComputer(
|
||||
api::UseComputerResult {
|
||||
result: Some(api::use_computer_result::Result::Success(
|
||||
@@ -1169,6 +1194,20 @@ impl TryFrom<UseComputerResult> for api::request::input::tool_call_result::Resul
|
||||
height: s.height as i32,
|
||||
}),
|
||||
cursor_position: result.cursor_position.map(vec_to_coordinates),
|
||||
windows: result
|
||||
.windows
|
||||
.into_iter()
|
||||
.map(convert_window_info)
|
||||
.collect(),
|
||||
// The window id is an opaque string on the wire; on macOS it is a
|
||||
// CGWindowID, so format the u32 back to a string at the boundary.
|
||||
captured_window: captured.map(|c| {
|
||||
api::use_computer_result::success::CapturedWindow {
|
||||
window_id: c.window_id.to_string(),
|
||||
width_px: c.width_px,
|
||||
height_px: c.height_px,
|
||||
}
|
||||
}),
|
||||
},
|
||||
)),
|
||||
},
|
||||
@@ -1205,6 +1244,18 @@ fn vec_to_coordinates(vec: computer_use::Vector2I) -> api::Coordinates {
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a computer_use window record into the API `WindowInfo` message.
|
||||
fn convert_window_info(window: computer_use::WindowInfo) -> api::WindowInfo {
|
||||
api::WindowInfo {
|
||||
// The window id travels as an opaque string; on macOS it is a CGWindowID (u32).
|
||||
window_id: window.window_id.to_string(),
|
||||
pid: window.pid,
|
||||
app_name: window.app_name,
|
||||
title: window.title,
|
||||
layer: window.layer,
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_platform(
|
||||
platform: computer_use::Platform,
|
||||
) -> api::request_computer_use_result::approved::Platform {
|
||||
@@ -1463,6 +1514,121 @@ impl From<AskUserQuestionResult> for api::request::input::tool_call_result::Resu
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RunAgentsLaunchedExecutionMode>
|
||||
for api::run_agents_result::launched::ResolvedExecutionMode
|
||||
{
|
||||
fn from(mode: RunAgentsLaunchedExecutionMode) -> Self {
|
||||
match mode {
|
||||
RunAgentsLaunchedExecutionMode::Local => {
|
||||
api::run_agents_result::launched::ResolvedExecutionMode::Local(
|
||||
api::run_agents::Local {},
|
||||
)
|
||||
}
|
||||
RunAgentsLaunchedExecutionMode::Remote {
|
||||
environment_id,
|
||||
worker_host,
|
||||
computer_use_enabled,
|
||||
} => api::run_agents_result::launched::ResolvedExecutionMode::Remote(
|
||||
api::run_agents::Remote {
|
||||
environment_id,
|
||||
worker_host,
|
||||
computer_use_enabled,
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RunAgentsAgentOutcome> for api::run_agents_result::AgentOutcome {
|
||||
fn from(outcome: RunAgentsAgentOutcome) -> Self {
|
||||
let result = match outcome.kind {
|
||||
RunAgentsAgentOutcomeKind::Launched { agent_id } => {
|
||||
api::run_agents_result::agent_outcome::Result::Launched(
|
||||
api::run_agents_result::LaunchedAgent { agent_id },
|
||||
)
|
||||
}
|
||||
RunAgentsAgentOutcomeKind::Failed { error } => {
|
||||
api::run_agents_result::agent_outcome::Result::Failed(
|
||||
api::run_agents_result::FailedAgent { error },
|
||||
)
|
||||
}
|
||||
};
|
||||
api::run_agents_result::AgentOutcome {
|
||||
name: outcome.name,
|
||||
result: Some(result),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps a client-side harness string identifier (e.g. "oz", "claude")
|
||||
/// to the new proto `Harness` oneof. Returns `None` for empty,
|
||||
/// unrecognized, or `"unknown"` strings; callers leave
|
||||
/// `resolved_harness` unset in that case.
|
||||
pub(super) fn build_api_harness(harness_type: &str) -> Option<api::Harness> {
|
||||
let normalized = harness_type.trim().to_ascii_lowercase().replace('_', "-");
|
||||
let variant = match normalized.as_str() {
|
||||
"oz" => api::harness::Variant::Oz(api::harness::Oz {}),
|
||||
"claude" | "claude-code" => api::harness::Variant::ClaudeCode(api::harness::ClaudeCode {}),
|
||||
"opencode" | "open-code" => api::harness::Variant::OpenCode(api::harness::OpenCode {}),
|
||||
"gemini" => api::harness::Variant::Gemini(api::harness::Gemini {}),
|
||||
"codex" => api::harness::Variant::Codex(api::harness::Codex {}),
|
||||
_ => return None,
|
||||
};
|
||||
Some(api::Harness {
|
||||
variant: Some(variant),
|
||||
})
|
||||
}
|
||||
|
||||
impl TryFrom<RunAgentsResult> for api::request::input::tool_call_result::Result {
|
||||
type Error = ConvertToAPITypeError;
|
||||
|
||||
fn try_from(result: RunAgentsResult) -> Result<Self, Self::Error> {
|
||||
match result {
|
||||
RunAgentsResult::Launched {
|
||||
model_id,
|
||||
harness_type,
|
||||
execution_mode,
|
||||
agents,
|
||||
} => Ok(
|
||||
api::request::input::tool_call_result::Result::RunAgentsResult(
|
||||
api::RunAgentsResult {
|
||||
outcome: Some(api::run_agents_result::Outcome::Launched(
|
||||
api::run_agents_result::Launched {
|
||||
resolved_model_id: model_id,
|
||||
resolved_harness: build_api_harness(&harness_type),
|
||||
resolved_execution_mode: Some(execution_mode.into()),
|
||||
agents: agents.into_iter().map(Into::into).collect(),
|
||||
},
|
||||
)),
|
||||
},
|
||||
),
|
||||
),
|
||||
RunAgentsResult::Denied { reason } => Ok(
|
||||
api::request::input::tool_call_result::Result::RunAgentsResult(
|
||||
api::RunAgentsResult {
|
||||
outcome: Some(api::run_agents_result::Outcome::Denied(
|
||||
api::run_agents_result::Denied { reason },
|
||||
)),
|
||||
},
|
||||
),
|
||||
),
|
||||
RunAgentsResult::Failure { error } => Ok(
|
||||
api::request::input::tool_call_result::Result::RunAgentsResult(
|
||||
api::RunAgentsResult {
|
||||
outcome: Some(api::run_agents_result::Outcome::Failure(
|
||||
api::run_agents_result::Failure { error },
|
||||
)),
|
||||
},
|
||||
),
|
||||
),
|
||||
// Reject is conveyed by the generic ToolCallResult.Cancel marker
|
||||
// synthesized server-side on the next user input; nothing for the
|
||||
// client to send on the wire here.
|
||||
RunAgentsResult::Cancelled => Err(ConvertToAPITypeError::Ignore),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<InsertReviewCommentsResult> for api::request::input::tool_call_result::Result {
|
||||
type Error = ConvertToAPITypeError;
|
||||
|
||||
@@ -1504,6 +1670,22 @@ impl TryFrom<InsertReviewCommentsResult> for api::request::input::tool_call_resu
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<WaitForEventsResult> for api::request::input::tool_call_result::Result {
|
||||
type Error = ConvertToAPITypeError;
|
||||
|
||||
/// Completed → wire form; Cancelled → drop (mirrors RunAgents).
|
||||
fn try_from(result: WaitForEventsResult) -> Result<Self, Self::Error> {
|
||||
match result {
|
||||
WaitForEventsResult::Completed => Ok(
|
||||
api::request::input::tool_call_result::Result::WaitForEvents(
|
||||
api::WaitForEventsResult {},
|
||||
),
|
||||
),
|
||||
WaitForEventsResult::Cancelled => Err(ConvertToAPITypeError::Ignore),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "convert_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
mod convert;
|
||||
|
||||
use std::{fmt::Display, ops::Range, time::SystemTime};
|
||||
use std::fmt::Display;
|
||||
use std::ops::Range;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use galaxy_core::command::ExitCode;
|
||||
use galaxy_terminal::model::BlockId;
|
||||
use chrono::{DateTime, Local};
|
||||
use itertools::Itertools as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use warp_multi_agent_api::apply_file_diffs_result::success::UpdatedFileContent;
|
||||
|
||||
use crate::{
|
||||
agent::FileLocations,
|
||||
document::{AIDocumentId, AIDocumentVersion},
|
||||
};
|
||||
use crate::agent::FileLocations;
|
||||
use crate::document::{AIDocumentId, AIDocumentVersion};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum AIAgentActionResultType {
|
||||
@@ -95,6 +94,14 @@ pub enum AIAgentActionResultType {
|
||||
TransferShellCommandControlToUser(TransferShellCommandControlToUserResult),
|
||||
/// The result of asking the user a question.
|
||||
AskUserQuestion(AskUserQuestionResult),
|
||||
|
||||
/// The result of an orchestrate tool call: launched (with per-agent
|
||||
/// outcomes), launch denied (Stage 2), failure, or cancelled.
|
||||
RunAgents(RunAgentsResult),
|
||||
|
||||
/// Result of the client-side wait_for_events watchdog or inbound
|
||||
/// resume.
|
||||
WaitForEvents(WaitForEventsResult),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
@@ -161,6 +168,8 @@ impl Display for AIAgentActionResultType {
|
||||
AIAgentActionResultType::SendMessageToAgent(result) => result.fmt(f),
|
||||
AIAgentActionResultType::TransferShellCommandControlToUser(result) => result.fmt(f),
|
||||
AIAgentActionResultType::AskUserQuestion(result) => result.fmt(f),
|
||||
AIAgentActionResultType::RunAgents(result) => result.fmt(f),
|
||||
AIAgentActionResultType::WaitForEvents(result) => result.fmt(f),
|
||||
AIAgentActionResultType::OpenCodeReview | AIAgentActionResultType::InitProject => {
|
||||
Ok(())
|
||||
}
|
||||
@@ -175,6 +184,8 @@ pub enum RequestCommandOutputResult {
|
||||
command: String,
|
||||
output: String,
|
||||
exit_code: ExitCode,
|
||||
start_ts: Option<DateTime<Local>>,
|
||||
completed_ts: Option<DateTime<Local>>,
|
||||
},
|
||||
LongRunningCommandSnapshot {
|
||||
block_id: BlockId,
|
||||
@@ -266,6 +277,8 @@ pub enum WriteToLongRunningShellCommandResult {
|
||||
block_id: BlockId,
|
||||
output: String,
|
||||
exit_code: ExitCode,
|
||||
start_ts: Option<DateTime<Local>>,
|
||||
completed_ts: Option<DateTime<Local>>,
|
||||
},
|
||||
Cancelled,
|
||||
Error(ShellCommandError),
|
||||
@@ -555,6 +568,8 @@ pub enum ReadShellCommandOutputResult {
|
||||
block_id: BlockId,
|
||||
output: String,
|
||||
exit_code: ExitCode,
|
||||
start_ts: Option<DateTime<Local>>,
|
||||
completed_ts: Option<DateTime<Local>>,
|
||||
},
|
||||
LongRunningCommandSnapshot {
|
||||
command: String,
|
||||
@@ -763,6 +778,12 @@ impl AIAgentActionResultType {
|
||||
AIAgentActionResultType::AskUserQuestion(_) => {
|
||||
"The user's answers to clarifying questions"
|
||||
}
|
||||
AIAgentActionResultType::RunAgents(_) => {
|
||||
"The result of an orchestrate batch of child agents"
|
||||
}
|
||||
AIAgentActionResultType::WaitForEvents(_) => {
|
||||
"The local watchdog timed out while waiting for inbound events"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -800,6 +821,8 @@ impl AIAgentActionResultType {
|
||||
| TransferShellCommandControlToUserResult::CommandFinished { .. },
|
||||
) => true,
|
||||
Self::AskUserQuestion(AskUserQuestionResult::Success { .. }) => true,
|
||||
Self::RunAgents(RunAgentsResult::Launched { .. }) => true,
|
||||
Self::WaitForEvents(WaitForEventsResult::Completed) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -828,7 +851,10 @@ impl AIAgentActionResultType {
|
||||
| Self::AskUserQuestion(AskUserQuestionResult::Error(_))
|
||||
| Self::TransferShellCommandControlToUser(
|
||||
TransferShellCommandControlToUserResult::Error(_),
|
||||
) => true,
|
||||
)
|
||||
| Self::RunAgents(RunAgentsResult::Failure { .. } | RunAgentsResult::Denied { .. }) => {
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -952,7 +978,9 @@ impl AIAgentActionResultType {
|
||||
| Self::StartAgent(StartAgentResult::Cancelled { .. })
|
||||
| Self::SendMessageToAgent(SendMessageToAgentResult::Cancelled)
|
||||
// SkippedByAutoApprove is intentionally excluded: the agent should continue.
|
||||
| Self::AskUserQuestion(AskUserQuestionResult::Cancelled) => true,
|
||||
| Self::AskUserQuestion(AskUserQuestionResult::Cancelled)
|
||||
| Self::RunAgents(RunAgentsResult::Cancelled)
|
||||
| Self::WaitForEvents(WaitForEventsResult::Cancelled) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1234,6 +1262,8 @@ pub enum RequestComputerUseResult {
|
||||
Approved {
|
||||
screenshot: computer_use::Screenshot,
|
||||
platform: computer_use::Platform,
|
||||
/// The on-screen windows the agent may target.
|
||||
windows: Vec<computer_use::WindowInfo>,
|
||||
},
|
||||
/// Request errored.
|
||||
Error(String),
|
||||
@@ -1280,6 +1310,9 @@ impl Display for FetchConversationResult {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(QUALITY-788): Delete legacy start_agent/start_agent_v2 result support once
|
||||
// old preview orchestration history no longer needs parse/display/result compatibility.
|
||||
// Linear issue: QUALITY-788.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum StartAgentResult {
|
||||
Success {
|
||||
@@ -1321,6 +1354,83 @@ impl Display for StartAgentResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// The terminal outcome of an orchestrate tool call.
|
||||
///
|
||||
/// Mirrors the proto `RunAgentsResult` oneof, with an additional
|
||||
/// `Cancelled` variant used internally by the action machinery when the
|
||||
/// user clicks Reject. The proto wire form for cancellation is the
|
||||
/// generic `ToolCallResult.Cancel` marker; the conversion code emits
|
||||
/// `ConvertToAPITypeError::Ignore` for `Cancelled` so the input
|
||||
/// interceptor can synthesize the marker on the next outbound input.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum RunAgentsResult {
|
||||
/// Orchestration launched. Carries the resolved configuration and one
|
||||
/// `AgentOutcome` per `agent_run_configs[]` entry, in input order.
|
||||
Launched {
|
||||
model_id: String,
|
||||
harness_type: String,
|
||||
execution_mode: RunAgentsLaunchedExecutionMode,
|
||||
agents: Vec<RunAgentsAgentOutcome>,
|
||||
},
|
||||
/// Declined for a non-error reason (currently disapproval).
|
||||
Denied { reason: String },
|
||||
/// Actual error path: server-side validation rejected the call, or the
|
||||
/// client could not begin the launch sequence at all.
|
||||
Failure { error: String },
|
||||
/// User rejected via the Reject button. Wire form is the generic
|
||||
/// `ToolCallResult.Cancel` marker, synthesized by the server's input
|
||||
/// interceptor on the next user input.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum RunAgentsLaunchedExecutionMode {
|
||||
Local,
|
||||
Remote {
|
||||
environment_id: String,
|
||||
worker_host: String,
|
||||
computer_use_enabled: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Per-agent outcome reported in `RunAgentsResult::Launched.agents`.
|
||||
/// Order mirrors the input order of `RunAgents.agent_run_configs[]`,
|
||||
/// regardless of which `CreateAgentTask` call returned first.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct RunAgentsAgentOutcome {
|
||||
pub name: String,
|
||||
pub kind: RunAgentsAgentOutcomeKind,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum RunAgentsAgentOutcomeKind {
|
||||
Launched { agent_id: String },
|
||||
Failed { error: String },
|
||||
}
|
||||
|
||||
impl Display for RunAgentsResult {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
RunAgentsResult::Launched { agents, .. } => {
|
||||
let launched = agents
|
||||
.iter()
|
||||
.filter(|a| matches!(a.kind, RunAgentsAgentOutcomeKind::Launched { .. }))
|
||||
.count();
|
||||
write!(
|
||||
f,
|
||||
"Orchestrate launched ({launched}/{} agents started)",
|
||||
agents.len()
|
||||
)
|
||||
}
|
||||
RunAgentsResult::Denied { reason } => {
|
||||
write!(f, "Orchestrate launch denied: {reason}")
|
||||
}
|
||||
RunAgentsResult::Failure { error } => write!(f, "Orchestrate failure: {error}"),
|
||||
RunAgentsResult::Cancelled => write!(f, "Orchestrate cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum SendMessageToAgentResult {
|
||||
Success { message_id: String },
|
||||
@@ -1353,6 +1463,8 @@ pub enum TransferShellCommandControlToUserResult {
|
||||
block_id: BlockId,
|
||||
output: String,
|
||||
exit_code: ExitCode,
|
||||
start_ts: Option<DateTime<Local>>,
|
||||
completed_ts: Option<DateTime<Local>>,
|
||||
},
|
||||
Cancelled,
|
||||
Error(ShellCommandError),
|
||||
@@ -1447,3 +1559,24 @@ impl Display for AskUserQuestionResult {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a client-side wait_for_events action.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum WaitForEventsResult {
|
||||
/// Watchdog fired or an inbound resume signal closed the wait. The
|
||||
/// agent's next turn observes an empty WaitForEvents result on the
|
||||
/// wire and decides how to proceed.
|
||||
Completed,
|
||||
/// User cancelled the conversation while waiting. Mirrors
|
||||
/// RunAgents::Cancelled: no tool-call result is sent on the wire.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl Display for WaitForEventsResult {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Completed => write!(f, "Wait for events completed"),
|
||||
Self::Cancelled => write!(f, "Wait for events cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,9 +5,22 @@ use warp_multi_agent_api as api;
|
||||
/// A citation listed in an AI response.
|
||||
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
|
||||
pub enum AIAgentCitation {
|
||||
WarpDriveObject { uid: String },
|
||||
WarpDocumentation { path: String },
|
||||
WebPage { url: String },
|
||||
WarpDriveObject {
|
||||
uid: String,
|
||||
},
|
||||
WarpDocumentation {
|
||||
path: String,
|
||||
},
|
||||
WebPage {
|
||||
url: String,
|
||||
},
|
||||
/// A memory from an attached memory store. `content` is the raw memory
|
||||
/// text shown as a preview in the chip.
|
||||
AgentMemory {
|
||||
memory_store_id: String,
|
||||
memory_id: String,
|
||||
content: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Display for AIAgentCitation {
|
||||
@@ -22,6 +35,13 @@ impl Display for AIAgentCitation {
|
||||
AIAgentCitation::WebPage { url } => {
|
||||
write!(f, "Web Page: {url}")
|
||||
}
|
||||
AIAgentCitation::AgentMemory {
|
||||
memory_store_id,
|
||||
memory_id,
|
||||
..
|
||||
} => {
|
||||
write!(f, "Agent Memory: {memory_store_id}/{memory_id}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,8 @@ use galaxy_terminal::shell::ShellLaunchData;
|
||||
use itertools::Itertools as _;
|
||||
|
||||
use crate::agent::action_result::FileContext;
|
||||
use crate::{index::locations::CodeContextLocation, paths::shell_native_absolute_path};
|
||||
use crate::index::locations::CodeContextLocation;
|
||||
use crate::paths::shell_native_absolute_path;
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct FileLocations {
|
||||
|
||||
@@ -3,6 +3,7 @@ pub mod action_result;
|
||||
mod citation;
|
||||
pub mod convert;
|
||||
pub mod file_locations;
|
||||
pub mod orchestration_config;
|
||||
|
||||
pub use citation::{AIAgentCitation, UnknownCitationTypeError};
|
||||
pub use file_locations::{group_file_contexts_for_display, FileLocations};
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
use warp_multi_agent_api as api;
|
||||
|
||||
use super::action::RunAgentsRequest;
|
||||
|
||||
/// Client-side representation of the orchestration config attached to a
|
||||
/// conversation via `OrchestrationConfigSnapshot`.
|
||||
///
|
||||
/// Mirrors the proto `OrchestrationConfig` but uses Rust-native types
|
||||
/// to keep view / model code free of proto imports.
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct OrchestrationConfig {
|
||||
pub model_id: String,
|
||||
pub harness_type: String,
|
||||
pub execution_mode: OrchestrationExecutionMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub enum OrchestrationExecutionMode {
|
||||
Local,
|
||||
Remote {
|
||||
environment_id: String,
|
||||
worker_host: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl OrchestrationExecutionMode {
|
||||
pub fn is_remote(&self) -> bool {
|
||||
matches!(self, Self::Remote { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// User's approval state for orchestration on the active config.
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)]
|
||||
pub enum OrchestrationConfigStatus {
|
||||
/// No `OrchestrationConfigSnapshot` has been seen yet.
|
||||
#[default]
|
||||
None,
|
||||
Approved,
|
||||
Disapproved,
|
||||
}
|
||||
|
||||
impl OrchestrationConfigStatus {
|
||||
pub fn is_approved(&self) -> bool {
|
||||
matches!(self, Self::Approved)
|
||||
}
|
||||
|
||||
pub fn is_disapproved(&self) -> bool {
|
||||
matches!(self, Self::Disapproved)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Match check — determines whether a `run_agents` call auto-launches.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns `true` when the `run_agents` call's run-wide fields match
|
||||
/// the active approved `OrchestrationConfig`, meaning the confirmation
|
||||
/// card can be skipped (auto-launch).
|
||||
///
|
||||
/// Empty/unset fields on the call are treated as inheriting from the
|
||||
/// config (and therefore matching). Fields not in the config
|
||||
/// (`computer_use_enabled`, `skills`, `base_prompt`, `agent_run_configs`,
|
||||
/// per-agent `title`) are excluded from the check.
|
||||
pub fn matches_active_config(request: &RunAgentsRequest, config: &OrchestrationConfig) -> bool {
|
||||
// model_id — empty on the call means "inherit from config" → matches.
|
||||
if !request.model_id.is_empty() && request.model_id != config.model_id {
|
||||
return false;
|
||||
}
|
||||
|
||||
// harness_type
|
||||
if !request.harness_type.is_empty() && request.harness_type != config.harness_type {
|
||||
return false;
|
||||
}
|
||||
|
||||
// execution_mode variant must agree.
|
||||
match (&request.execution_mode, &config.execution_mode) {
|
||||
(super::action::RunAgentsExecutionMode::Local, OrchestrationExecutionMode::Local) => true,
|
||||
(
|
||||
super::action::RunAgentsExecutionMode::Remote {
|
||||
environment_id,
|
||||
worker_host,
|
||||
..
|
||||
},
|
||||
OrchestrationExecutionMode::Remote {
|
||||
environment_id: cfg_env,
|
||||
worker_host: cfg_host,
|
||||
},
|
||||
) => {
|
||||
let env_matches = environment_id.is_empty() || environment_id == cfg_env;
|
||||
let host_matches = worker_host.is_empty() || worker_host == cfg_host;
|
||||
env_matches && host_matches
|
||||
}
|
||||
// Variant mismatch (Local vs Remote).
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Proto ↔ native conversions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
impl OrchestrationConfig {
|
||||
/// Converts from the proto `OrchestrationConfig` message.
|
||||
pub fn from_proto(proto: &api::OrchestrationConfig) -> Self {
|
||||
let execution_mode = match &proto.execution_mode {
|
||||
Some(api::orchestration_config::ExecutionMode::Remote(remote)) => {
|
||||
OrchestrationExecutionMode::Remote {
|
||||
environment_id: remote.environment_id.clone(),
|
||||
worker_host: remote.worker_host.clone(),
|
||||
}
|
||||
}
|
||||
Some(api::orchestration_config::ExecutionMode::Local(_)) | None => {
|
||||
OrchestrationExecutionMode::Local
|
||||
}
|
||||
};
|
||||
|
||||
let harness_type = harness_proto_to_string(proto.harness.as_ref()).unwrap_or_default();
|
||||
|
||||
Self {
|
||||
model_id: proto.model_id.clone(),
|
||||
harness_type,
|
||||
execution_mode,
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts to the proto `OrchestrationConfig` message.
|
||||
pub fn to_proto(&self) -> api::OrchestrationConfig {
|
||||
let execution_mode = match &self.execution_mode {
|
||||
OrchestrationExecutionMode::Local => {
|
||||
Some(api::orchestration_config::ExecutionMode::Local(
|
||||
api::orchestration_config::Local {},
|
||||
))
|
||||
}
|
||||
OrchestrationExecutionMode::Remote {
|
||||
environment_id,
|
||||
worker_host,
|
||||
} => Some(api::orchestration_config::ExecutionMode::Remote(
|
||||
api::orchestration_config::Remote {
|
||||
environment_id: environment_id.clone(),
|
||||
worker_host: worker_host.clone(),
|
||||
},
|
||||
)),
|
||||
};
|
||||
|
||||
api::OrchestrationConfig {
|
||||
model_id: self.model_id.clone(),
|
||||
harness: harness_type_to_proto(&self.harness_type),
|
||||
execution_mode,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl OrchestrationConfigStatus {
|
||||
/// Converts from the proto `OrchestrationStatus` message.
|
||||
pub fn from_proto(proto: Option<&api::OrchestrationStatus>) -> Self {
|
||||
let Some(status) = proto else {
|
||||
return Self::None;
|
||||
};
|
||||
match &status.status {
|
||||
Some(api::orchestration_status::Status::Approved(_)) => Self::Approved,
|
||||
Some(api::orchestration_status::Status::Disapproved(_)) => Self::Disapproved,
|
||||
None => Self::None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts to the proto `OrchestrationStatus` message.
|
||||
pub fn to_proto(&self) -> Option<api::OrchestrationStatus> {
|
||||
match self {
|
||||
Self::None => None,
|
||||
Self::Approved => Some(api::OrchestrationStatus {
|
||||
status: Some(api::orchestration_status::Status::Approved(
|
||||
api::orchestration_status::Approved {},
|
||||
)),
|
||||
}),
|
||||
Self::Disapproved => Some(api::OrchestrationStatus {
|
||||
status: Some(api::orchestration_status::Status::Disapproved(
|
||||
api::orchestration_status::Disapproved {},
|
||||
)),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps the proto `Harness` oneof to a client-side string identifier.
|
||||
/// Returns `None` for an unset variant.
|
||||
fn harness_proto_to_string(harness: Option<&api::Harness>) -> Option<String> {
|
||||
let variant = harness?.variant.as_ref()?;
|
||||
Some(
|
||||
match variant {
|
||||
api::harness::Variant::Oz(_) => "oz",
|
||||
api::harness::Variant::ClaudeCode(_) => "claude",
|
||||
api::harness::Variant::OpenCode(_) => "opencode",
|
||||
api::harness::Variant::Gemini(_) => "gemini",
|
||||
api::harness::Variant::Codex(_) => "codex",
|
||||
}
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Converts a client-side harness string identifier to the proto `Harness`
|
||||
/// oneof variant. Returns `None` for empty or unknown strings.
|
||||
fn harness_type_to_proto(harness_type: &str) -> Option<api::Harness> {
|
||||
let variant = match harness_type {
|
||||
"oz" => api::harness::Variant::Oz(api::harness::Oz {}),
|
||||
"claude" => api::harness::Variant::ClaudeCode(api::harness::ClaudeCode {}),
|
||||
"opencode" => api::harness::Variant::OpenCode(api::harness::OpenCode {}),
|
||||
"gemini" => api::harness::Variant::Gemini(api::harness::Gemini {}),
|
||||
"codex" => api::harness::Variant::Codex(api::harness::Codex {}),
|
||||
_ => return None,
|
||||
};
|
||||
Some(api::Harness {
|
||||
variant: Some(variant),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "orchestration_config_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,180 @@
|
||||
use super::*;
|
||||
use crate::agent::action::{RunAgentsAgentRunConfig, RunAgentsExecutionMode, RunAgentsRequest};
|
||||
|
||||
fn make_config(model: &str, harness: &str, remote: bool) -> OrchestrationConfig {
|
||||
OrchestrationConfig {
|
||||
model_id: model.to_string(),
|
||||
harness_type: harness.to_string(),
|
||||
execution_mode: if remote {
|
||||
OrchestrationExecutionMode::Remote {
|
||||
environment_id: "env-1".to_string(),
|
||||
worker_host: "warp".to_string(),
|
||||
}
|
||||
} else {
|
||||
OrchestrationExecutionMode::Local
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn make_request(model: &str, harness: &str, remote: bool) -> RunAgentsRequest {
|
||||
RunAgentsRequest {
|
||||
summary: "test".to_string(),
|
||||
base_prompt: "prompt".to_string(),
|
||||
skills: vec![],
|
||||
model_id: model.to_string(),
|
||||
harness_type: harness.to_string(),
|
||||
execution_mode: if remote {
|
||||
RunAgentsExecutionMode::Remote {
|
||||
environment_id: "env-1".to_string(),
|
||||
worker_host: "warp".to_string(),
|
||||
computer_use_enabled: false,
|
||||
}
|
||||
} else {
|
||||
RunAgentsExecutionMode::Local
|
||||
},
|
||||
agent_run_configs: vec![RunAgentsAgentRunConfig {
|
||||
name: "a".to_string(),
|
||||
prompt: String::new(),
|
||||
title: String::new(),
|
||||
}],
|
||||
plan_id: String::new(),
|
||||
harness_auth_secret_name: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_match_local() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("auto", "oz", false);
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_match_remote() {
|
||||
let config = make_config("auto", "oz", true);
|
||||
let request = make_request("auto", "oz", true);
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_model_inherits_and_matches() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("", "oz", false);
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_harness_inherits_and_matches() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("auto", "", false);
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_model_mismatches() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("claude-4-6-opus-high", "oz", false);
|
||||
assert!(!matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_harness_mismatches() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("auto", "claude", false);
|
||||
assert!(!matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_mode_variant_mismatch() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let request = make_request("auto", "oz", true);
|
||||
assert!(!matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_different_environment_mismatches() {
|
||||
let config = make_config("auto", "oz", true);
|
||||
let mut request = make_request("auto", "oz", true);
|
||||
if let RunAgentsExecutionMode::Remote {
|
||||
ref mut environment_id,
|
||||
..
|
||||
} = request.execution_mode
|
||||
{
|
||||
*environment_id = "env-other".to_string();
|
||||
}
|
||||
assert!(!matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_empty_env_inherits_and_matches() {
|
||||
let config = make_config("auto", "oz", true);
|
||||
let mut request = make_request("auto", "oz", true);
|
||||
if let RunAgentsExecutionMode::Remote {
|
||||
ref mut environment_id,
|
||||
..
|
||||
} = request.execution_mode
|
||||
{
|
||||
*environment_id = String::new();
|
||||
}
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn computer_use_not_in_match_check() {
|
||||
let config = make_config("auto", "oz", true);
|
||||
let mut request = make_request("auto", "oz", true);
|
||||
if let RunAgentsExecutionMode::Remote {
|
||||
ref mut computer_use_enabled,
|
||||
..
|
||||
} = request.execution_mode
|
||||
{
|
||||
*computer_use_enabled = true;
|
||||
}
|
||||
// computer_use_enabled differs but should still match
|
||||
assert!(matches_active_config(&request, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_default_is_none() {
|
||||
assert_eq!(
|
||||
OrchestrationConfigStatus::default(),
|
||||
OrchestrationConfigStatus::None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_predicates() {
|
||||
assert!(OrchestrationConfigStatus::Approved.is_approved());
|
||||
assert!(!OrchestrationConfigStatus::Approved.is_disapproved());
|
||||
assert!(OrchestrationConfigStatus::Disapproved.is_disapproved());
|
||||
assert!(!OrchestrationConfigStatus::None.is_approved());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proto_round_trip_config_local() {
|
||||
let config = make_config("auto", "oz", false);
|
||||
let proto = config.to_proto();
|
||||
let round_tripped = OrchestrationConfig::from_proto(&proto);
|
||||
assert_eq!(config, round_tripped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proto_round_trip_config_remote() {
|
||||
let config = make_config("auto", "claude", true);
|
||||
let proto = config.to_proto();
|
||||
let round_tripped = OrchestrationConfig::from_proto(&proto);
|
||||
assert_eq!(config, round_tripped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proto_round_trip_status() {
|
||||
for status in [
|
||||
OrchestrationConfigStatus::None,
|
||||
OrchestrationConfigStatus::Approved,
|
||||
OrchestrationConfigStatus::Disapproved,
|
||||
] {
|
||||
let proto = status.to_proto();
|
||||
let round_tripped = OrchestrationConfigStatus::from_proto(proto.as_ref());
|
||||
assert_eq!(status, round_tripped);
|
||||
}
|
||||
}
|
||||
+437
-13
@@ -1,11 +1,24 @@
|
||||
pub use crate::aws_credentials::{AwsCredentials, AwsCredentialsState};
|
||||
use galaxyui::{Entity, ModelContext, SingletonEntity};
|
||||
use galaxyui_extras::secure_storage::{self, AppContextExt};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
use warp_multi_agent_api as api;
|
||||
use galaxyui_core::{Entity, ModelContext, SingletonEntity};
|
||||
use galaxyui_extras::secure_storage::{self, AppContextExt};
|
||||
|
||||
pub use crate::aws_credentials::{AwsCredentials, AwsCredentialsState};
|
||||
pub use crate::geap_credentials::{
|
||||
GeapCredentials, GeapCredentialsState, GeapFederation, GeapMintBinding,
|
||||
LoadGeapCredentialsError, GEAP_REFRESH_LEAD_TIME,
|
||||
};
|
||||
|
||||
const SECURE_STORAGE_KEY: &str = "AiApiKeys";
|
||||
|
||||
/// Secure-storage key for the connected xAI/Grok subscription's OAuth tokens.
|
||||
/// Kept separate from [`SECURE_STORAGE_KEY`] because these are OAuth tokens with
|
||||
/// a refresh lifecycle, not a user-pasted static key.
|
||||
const GROK_SECURE_STORAGE_KEY: &str = "GrokOAuthTokens";
|
||||
|
||||
/// Emitted when user-provided API keys are updated in-memory.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ApiKeyManagerEvent {
|
||||
@@ -17,11 +30,44 @@ pub enum ApiKeyManagerEvent {
|
||||
/// These are used for "Bring Your Own API Key" functionality, allowing
|
||||
/// users to use their own API keys instead of Warp's.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ApiKeys {
|
||||
pub google: Option<String>,
|
||||
pub anthropic: Option<String>,
|
||||
pub openai: Option<String>,
|
||||
pub open_router: Option<String>,
|
||||
pub custom_endpoints: Vec<CustomEndpoint>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct CustomEndpoint {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
pub api_key: String,
|
||||
pub models: Vec<CustomEndpointModel>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct CustomEndpointModel {
|
||||
pub name: String,
|
||||
pub alias: Option<String>,
|
||||
/// Stable identifier used as `ModelConfig.{base,coding,cli_agent,computer_use_agent}` and
|
||||
/// as the `CustomModelProviders.providers[*].models[*].config_key` on the request wire.
|
||||
/// Generated as a UUIDv4 at model creation.
|
||||
pub config_key: String,
|
||||
}
|
||||
|
||||
impl CustomEndpointModel {
|
||||
/// Picker label: prefer the user-provided alias; fall back to the raw model name
|
||||
/// so a row is never blank.
|
||||
pub fn display_label(&self) -> &str {
|
||||
match self.alias.as_deref() {
|
||||
Some(alias) if !alias.trim().is_empty() => alias,
|
||||
_ => &self.name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiKeys {
|
||||
@@ -30,6 +76,70 @@ impl ApiKeys {
|
||||
|| self.anthropic.is_some()
|
||||
|| self.google.is_some()
|
||||
|| self.open_router.is_some()
|
||||
|| self
|
||||
.custom_endpoints
|
||||
.iter()
|
||||
.any(|endpoint| !endpoint.api_key.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Number of single-provider API keys currently configured (OpenAI,
|
||||
/// Anthropic, Google, OpenRouter). Custom endpoints are counted separately
|
||||
/// via `custom_endpoints`.
|
||||
pub fn provider_key_count(&self) -> usize {
|
||||
[
|
||||
&self.openai,
|
||||
&self.anthropic,
|
||||
&self.google,
|
||||
&self.open_router,
|
||||
]
|
||||
.into_iter()
|
||||
.filter(|key| key.as_deref().is_some_and(|v| !v.trim().is_empty()))
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
/// OAuth tokens for a connected xAI / Grok subscription (e.g. SuperGrok).
|
||||
///
|
||||
/// Persisted to secure storage under [`GROK_SECURE_STORAGE_KEY`], separate from
|
||||
/// the BYO [`ApiKeys`] blob because these are OAuth tokens with a refresh
|
||||
/// lifecycle rather than a user-pasted static key. `crate::grok_subscription`
|
||||
/// owns refreshing them; this module is the storage and request-injection
|
||||
/// source of truth that [`ApiKeyManager::api_keys_for_request`] reads from.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub struct GrokTokens {
|
||||
pub access_token: String,
|
||||
#[serde(default)]
|
||||
pub refresh_token: Option<String>,
|
||||
/// Absolute time at which `access_token` expires, if the provider told us.
|
||||
#[serde(default)]
|
||||
pub expires_at: Option<SystemTime>,
|
||||
/// When the user originally connected the subscription (i.e. when the
|
||||
/// browser OAuth flow completed). Carried over across token refreshes so
|
||||
/// it keeps reflecting the initial connection, not the latest refresh;
|
||||
/// surfaced in the settings UI as "Connected on ...". `None` for tokens
|
||||
/// stored before this field existed.
|
||||
#[serde(default)]
|
||||
pub connected_at: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl GrokTokens {
|
||||
/// Returns the access token whenever it is non-empty, regardless of
|
||||
/// expiry. Possibly-expired tokens are still sent so the server stays the
|
||||
/// final authority on token validity (it rejects truly invalid tokens);
|
||||
/// `crate::grok_subscription` refreshes (nearly) expired tokens in the
|
||||
/// background.
|
||||
pub fn access_token_for_request(&self) -> Option<&str> {
|
||||
(!self.access_token.trim().is_empty()).then_some(self.access_token.as_str())
|
||||
}
|
||||
|
||||
/// Returns `true` when the token is known to expire within `lead_time` and
|
||||
/// should be proactively refreshed. Tokens with an unknown expiry never
|
||||
/// report as needing a refresh (there's no expiry signal to act on).
|
||||
pub fn needs_refresh(&self, lead_time: Duration) -> bool {
|
||||
match self.expires_at {
|
||||
Some(expires_at) => expires_at <= SystemTime::now() + lead_time,
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,27 +151,55 @@ pub enum AwsCredentialsRefreshStrategy {
|
||||
LocalChain,
|
||||
/// Credentials are managed externally via OIDC/STS.
|
||||
/// The task ID is used to scope the STS AssumeRoleWithWebIdentity session.
|
||||
/// The role ARN is the IAM role to assume via STS.
|
||||
/// The role ARN + region are the info used to assume the IAM role via STS.
|
||||
OidcManaged {
|
||||
task_id: Option<String>,
|
||||
role_arn: String,
|
||||
region: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// A structure that manages API keys for AI providers.
|
||||
pub struct ApiKeyManager {
|
||||
keys: ApiKeys,
|
||||
/// OAuth tokens for a connected xAI/Grok subscription, if any. Persisted
|
||||
/// separately from `keys` under [`GROK_SECURE_STORAGE_KEY`];
|
||||
/// `crate::grok_subscription` keeps these fresh.
|
||||
grok_tokens: Option<GrokTokens>,
|
||||
/// Whether background refresh of `grok_tokens` is currently allowed.
|
||||
/// Mirrors the BYO API key policy, which lives in the app layer; wired in
|
||||
/// via `ApiKeyManager::set_grok_refresh_allowed` (`crate::grok_subscription`).
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub(crate) grok_refresh_allowed: bool,
|
||||
/// Guards against overlapping Grok token refreshes: the proactive refresh
|
||||
/// timer and the request-time safety net
|
||||
/// (`ApiKeyManager::refresh_grok_tokens_if_needed`) can otherwise race.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub(crate) grok_refresh_in_flight: bool,
|
||||
pub(crate) aws_credentials_state: AwsCredentialsState,
|
||||
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy,
|
||||
/// In-memory Gemini Enterprise (GEAP) credential state.
|
||||
pub(crate) geap_credentials_state: GeapCredentialsState,
|
||||
secure_storage_write_version: u64,
|
||||
grok_secure_storage_write_version: u64,
|
||||
}
|
||||
|
||||
impl ApiKeyManager {
|
||||
pub fn new(ctx: &mut ModelContext<Self>) -> Self {
|
||||
let keys = Self::load_keys_from_secure_storage(ctx);
|
||||
let grok_tokens = Self::load_grok_tokens_from_secure_storage(ctx);
|
||||
Self {
|
||||
keys,
|
||||
grok_tokens,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
grok_refresh_allowed: false,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
grok_refresh_in_flight: false,
|
||||
aws_credentials_state: AwsCredentialsState::Missing,
|
||||
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy::default(),
|
||||
geap_credentials_state: GeapCredentialsState::Missing,
|
||||
secure_storage_write_version: 0,
|
||||
grok_secure_storage_write_version: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +207,39 @@ impl ApiKeyManager {
|
||||
&self.keys
|
||||
}
|
||||
|
||||
/// The currently stored xAI/Grok OAuth tokens, if the user has connected a
|
||||
/// Grok subscription.
|
||||
pub fn grok_tokens(&self) -> Option<&GrokTokens> {
|
||||
self.grok_tokens.as_ref()
|
||||
}
|
||||
|
||||
/// Returns `true` when a Grok subscription is connected with a usable OAuth
|
||||
/// access token.
|
||||
pub fn has_grok_subscription(&self) -> bool {
|
||||
self.grok_tokens
|
||||
.as_ref()
|
||||
.and_then(GrokTokens::access_token_for_request)
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// Returns `true` when the user has any usable BYO credential: a pasted
|
||||
/// provider or custom-endpoint key, or a connected Grok subscription.
|
||||
pub fn has_any_key(&self) -> bool {
|
||||
self.keys.has_any_key() || self.has_grok_subscription()
|
||||
}
|
||||
|
||||
/// Stores (or clears, with `None`) the xAI/Grok OAuth tokens and persists
|
||||
/// them to secure storage. No-op when the value is unchanged so we don't
|
||||
/// emit spurious events or schedule redundant keychain writes.
|
||||
pub fn set_grok_tokens(&mut self, tokens: Option<GrokTokens>, ctx: &mut ModelContext<Self>) {
|
||||
if self.grok_tokens == tokens {
|
||||
return;
|
||||
}
|
||||
self.grok_tokens = tokens;
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
self.write_grok_tokens_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn set_google_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
|
||||
self.keys.google = key;
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
@@ -93,6 +264,82 @@ impl ApiKeyManager {
|
||||
self.write_keys_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn add_custom_endpoint(
|
||||
&mut self,
|
||||
name: String,
|
||||
url: String,
|
||||
api_key: String,
|
||||
models: Vec<(String, Option<String>, Option<String>)>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
self.keys.custom_endpoints.push(CustomEndpoint {
|
||||
name,
|
||||
url,
|
||||
api_key,
|
||||
models: models
|
||||
.into_iter()
|
||||
.map(|(name, alias, config_key)| CustomEndpointModel {
|
||||
name,
|
||||
alias,
|
||||
config_key: config_key
|
||||
.filter(|k| !k.is_empty())
|
||||
.unwrap_or_else(|| Uuid::new_v4().to_string()),
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
self.write_keys_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn save_custom_endpoint(
|
||||
&mut self,
|
||||
index: usize,
|
||||
name: String,
|
||||
url: String,
|
||||
api_key: String,
|
||||
models: Vec<(String, Option<String>, Option<String>)>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
if index >= self.keys.custom_endpoints.len() {
|
||||
return;
|
||||
}
|
||||
self.keys.custom_endpoints[index] = CustomEndpoint {
|
||||
name,
|
||||
url,
|
||||
api_key,
|
||||
models: models
|
||||
.into_iter()
|
||||
.map(|(name, alias, config_key)| CustomEndpointModel {
|
||||
name,
|
||||
alias,
|
||||
config_key: config_key
|
||||
.filter(|k| !k.is_empty())
|
||||
.unwrap_or_else(|| Uuid::new_v4().to_string()),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
self.write_keys_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn remove_custom_endpoint(&mut self, index: usize, ctx: &mut ModelContext<Self>) {
|
||||
if index >= self.keys.custom_endpoints.len() {
|
||||
return;
|
||||
}
|
||||
self.keys.custom_endpoints.remove(index);
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
self.write_keys_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn clear_custom_endpoints(&mut self, ctx: &mut ModelContext<Self>) {
|
||||
if self.keys.custom_endpoints.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.keys.custom_endpoints.clear();
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
self.write_keys_to_secure_storage(ctx);
|
||||
}
|
||||
|
||||
pub fn set_aws_credentials_state(
|
||||
&mut self,
|
||||
state: AwsCredentialsState,
|
||||
@@ -106,6 +353,22 @@ impl ApiKeyManager {
|
||||
&self.aws_credentials_state
|
||||
}
|
||||
|
||||
pub fn set_geap_credentials_state(
|
||||
&mut self,
|
||||
state: GeapCredentialsState,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
if self.geap_credentials_state == state {
|
||||
return;
|
||||
}
|
||||
self.geap_credentials_state = state;
|
||||
ctx.emit(ApiKeyManagerEvent::KeysUpdated);
|
||||
}
|
||||
|
||||
pub fn geap_credentials_state(&self) -> &GeapCredentialsState {
|
||||
&self.geap_credentials_state
|
||||
}
|
||||
|
||||
pub fn aws_credentials_refresh_strategy(&self) -> AwsCredentialsRefreshStrategy {
|
||||
self.aws_credentials_refresh_strategy.clone()
|
||||
}
|
||||
@@ -117,10 +380,60 @@ impl ApiKeyManager {
|
||||
self.aws_credentials_refresh_strategy = strategy;
|
||||
}
|
||||
|
||||
/// Builds the `CustomModelProviders` registry that ships with every agent request.
|
||||
///
|
||||
/// Emits one [`CustomModelProvider`] per configured [`CustomEndpoint`], each populated with
|
||||
/// all of its [`CustomEndpointModel`]s. The per-model `config_key` is what the server uses
|
||||
/// to map a `ModelConfig.{base,coding,cli_agent,computer_use_agent}` selection back to a
|
||||
/// user-provided endpoint, so it MUST be the same UUID we store locally.
|
||||
///
|
||||
/// Returns `None` when custom models should not be included or no endpoint has both a
|
||||
/// non-empty URL and API key.
|
||||
pub fn custom_model_providers_for_request(
|
||||
&self,
|
||||
include_custom_models: bool,
|
||||
) -> Option<api::request::settings::CustomModelProviders> {
|
||||
if !include_custom_models {
|
||||
return None;
|
||||
}
|
||||
|
||||
let providers: Vec<_> = self
|
||||
.keys
|
||||
.custom_endpoints
|
||||
.iter()
|
||||
.filter(|endpoint| !endpoint.url.trim().is_empty() && !endpoint.api_key.is_empty())
|
||||
.map(
|
||||
|endpoint| api::request::settings::custom_model_providers::CustomModelProvider {
|
||||
base_url: endpoint.url.clone(),
|
||||
api_key: endpoint.api_key.clone(),
|
||||
models: endpoint
|
||||
.models
|
||||
.iter()
|
||||
.filter(|m| !m.name.trim().is_empty() && !m.config_key.is_empty())
|
||||
.map(
|
||||
|m| api::request::settings::custom_model_providers::CustomModel {
|
||||
slug: m.name.clone(),
|
||||
config_key: m.config_key.clone(),
|
||||
},
|
||||
)
|
||||
.collect(),
|
||||
},
|
||||
)
|
||||
.filter(|provider| !provider.models.is_empty())
|
||||
.collect();
|
||||
|
||||
if providers.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(api::request::settings::CustomModelProviders { providers })
|
||||
}
|
||||
}
|
||||
|
||||
pub fn api_keys_for_request(
|
||||
&self,
|
||||
include_byo_keys: bool,
|
||||
include_aws_bedrock_credentials: bool,
|
||||
geap_binding: Option<GeapMintBinding>,
|
||||
) -> Option<api::request::settings::ApiKeys> {
|
||||
let anthropic = include_byo_keys
|
||||
.then(|| self.keys.anthropic.clone())
|
||||
@@ -138,6 +451,22 @@ impl ApiKeyManager {
|
||||
.then(|| self.keys.open_router.clone())
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
|
||||
// The connected Grok subscription's OAuth access token is user-provided
|
||||
// auth, just like a pasted BYO API key, so it respects the same BYO
|
||||
// policy gate: when BYO keys are disabled (e.g. by workspace policy),
|
||||
// the token must not be sent. Possibly-expired tokens ARE sent — the
|
||||
// server is the authority on validity.
|
||||
let grok_oauth_access_token = include_byo_keys
|
||||
.then(|| {
|
||||
self.grok_tokens
|
||||
.as_ref()
|
||||
.and_then(GrokTokens::access_token_for_request)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
|
||||
// Also include credentials when running with OIDC-managed Bedrock inference, regardless
|
||||
// of the per-user setting flag (which only applies to the local credential chain path).
|
||||
let include_aws = include_aws_bedrock_credentials
|
||||
@@ -154,11 +483,36 @@ impl ApiKeyManager {
|
||||
})
|
||||
.flatten();
|
||||
|
||||
// Gemini Enterprise (GEAP) credentials attach only when the caller's
|
||||
// gate is on AND the stored token was minted for that same
|
||||
// (user, audience, SA) binding.
|
||||
let google_cloud_credentials: Option<
|
||||
api::request::settings::api_keys::GoogleCloudCredentials,
|
||||
> = geap_binding
|
||||
.as_ref()
|
||||
.and_then(|binding| match self.geap_credentials_state {
|
||||
GeapCredentialsState::Loaded {
|
||||
ref credentials,
|
||||
ref minted_for,
|
||||
..
|
||||
} if minted_for == binding => credentials
|
||||
.access_token_for_request()
|
||||
.map(|_| credentials.clone().into()),
|
||||
GeapCredentialsState::Refreshing {
|
||||
previous: Some((ref credentials, ref minted_for)),
|
||||
} if minted_for == binding => credentials
|
||||
.access_token_for_request()
|
||||
.map(|_| credentials.clone().into()),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
if anthropic.is_empty()
|
||||
&& openai.is_empty()
|
||||
&& google.is_empty()
|
||||
&& open_router.is_empty()
|
||||
&& grok_oauth_access_token.is_empty()
|
||||
&& aws_credentials.is_none()
|
||||
&& google_cloud_credentials.is_none()
|
||||
{
|
||||
None
|
||||
} else {
|
||||
@@ -167,8 +521,10 @@ impl ApiKeyManager {
|
||||
openai,
|
||||
google,
|
||||
open_router,
|
||||
grok_oauth_access_token,
|
||||
allow_use_of_warp_credits: false,
|
||||
aws_credentials,
|
||||
google_cloud_credentials,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -184,32 +540,96 @@ impl ApiKeyManager {
|
||||
}
|
||||
};
|
||||
|
||||
let keys = match serde_json::from_str(&key_json) {
|
||||
match serde_json::from_str(&key_json) {
|
||||
Ok(keys) => keys,
|
||||
Err(e) => {
|
||||
log::error!("Failed to deserialize API keys: {e:#}");
|
||||
ApiKeys::default()
|
||||
}
|
||||
};
|
||||
|
||||
keys
|
||||
}
|
||||
}
|
||||
|
||||
fn write_keys_to_secure_storage(&mut self, ctx: &mut ModelContext<Self>) {
|
||||
let keys = self.keys.clone();
|
||||
|
||||
let json = match serde_json::to_string(&keys) {
|
||||
let json = match serde_json::to_string(&self.keys) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
log::error!("Failed to serialize API keys: {e:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.secure_storage_write_version += 1;
|
||||
let write_version = self.secure_storage_write_version;
|
||||
|
||||
if let Err(e) = ctx.secure_storage().write_value(SECURE_STORAGE_KEY, &json) {
|
||||
log::error!("Failed to write API keys to secure storage: {e:#}");
|
||||
// Defer the keychain write so it doesn't block the current event
|
||||
// processing. The in-memory state is already updated and events
|
||||
// already emitted, so the UI updates immediately while the
|
||||
// potentially slow platform secure-storage call runs in a
|
||||
// subsequent main-thread callback. Skip stale callbacks so older
|
||||
// writes cannot complete after and overwrite a newer payload.
|
||||
ctx.spawn(async move { json }, move |me, json, ctx| {
|
||||
if write_version != me.secure_storage_write_version {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = ctx.secure_storage().write_value(SECURE_STORAGE_KEY, &json) {
|
||||
log::error!("Failed to write API keys to secure storage: {e:#}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn load_grok_tokens_from_secure_storage(ctx: &mut ModelContext<Self>) -> Option<GrokTokens> {
|
||||
let json = match ctx.secure_storage().read_value(GROK_SECURE_STORAGE_KEY) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
if !matches!(e, secure_storage::Error::NotFound) {
|
||||
log::error!("Failed to read Grok tokens from secure storage: {e:#}");
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match serde_json::from_str(&json) {
|
||||
Ok(tokens) => Some(tokens),
|
||||
Err(e) => {
|
||||
log::error!("Failed to deserialize Grok tokens: {e:#}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_grok_tokens_to_secure_storage(&mut self, ctx: &mut ModelContext<Self>) {
|
||||
// `Some(json)` writes the tokens; `None` removes the stored entry (the
|
||||
// user disconnected). Serialize up front so the deferred callback only
|
||||
// touches the keychain.
|
||||
let payload = match self.grok_tokens.as_ref().map(serde_json::to_string) {
|
||||
Some(Ok(json)) => Some(json),
|
||||
Some(Err(e)) => {
|
||||
log::error!("Failed to serialize Grok tokens: {e:#}");
|
||||
return;
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
self.grok_secure_storage_write_version += 1;
|
||||
let write_version = self.grok_secure_storage_write_version;
|
||||
|
||||
// Defer the keychain write/remove like `write_keys_to_secure_storage`,
|
||||
// skipping stale callbacks so an older write can't clobber a newer one.
|
||||
ctx.spawn(async move { payload }, move |me, payload, ctx| {
|
||||
if write_version != me.grok_secure_storage_write_version {
|
||||
return;
|
||||
}
|
||||
let result = match payload {
|
||||
Some(ref json) => ctx
|
||||
.secure_storage()
|
||||
.write_value(GROK_SECURE_STORAGE_KEY, json),
|
||||
None => ctx.secure_storage().remove_value(GROK_SECURE_STORAGE_KEY),
|
||||
};
|
||||
if let Err(e) = result {
|
||||
if !matches!(e, secure_storage::Error::NotFound) {
|
||||
log::error!("Failed to persist Grok tokens to secure storage: {e:#}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl Entity for ApiKeyManager {
|
||||
@@ -217,3 +637,7 @@ impl Entity for ApiKeyManager {
|
||||
}
|
||||
|
||||
impl SingletonEntity for ApiKeyManager {}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "api_keys_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,696 @@
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn make_manager(keys: ApiKeys) -> ApiKeyManager {
|
||||
make_manager_with_grok(keys, None)
|
||||
}
|
||||
|
||||
fn make_manager_with_grok(keys: ApiKeys, grok_tokens: Option<GrokTokens>) -> ApiKeyManager {
|
||||
ApiKeyManager {
|
||||
keys,
|
||||
grok_tokens,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
grok_refresh_allowed: false,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
grok_refresh_in_flight: false,
|
||||
aws_credentials_state: AwsCredentialsState::Missing,
|
||||
aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy::default(),
|
||||
geap_credentials_state: GeapCredentialsState::Missing,
|
||||
secure_storage_write_version: 0,
|
||||
grok_secure_storage_write_version: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_manager_with_geap(geap_credentials_state: GeapCredentialsState) -> ApiKeyManager {
|
||||
let mut manager = make_manager(ApiKeys::default());
|
||||
manager.geap_credentials_state = geap_credentials_state;
|
||||
manager
|
||||
}
|
||||
|
||||
fn grok_tokens(access_token: &str, expires_in: Option<u64>) -> GrokTokens {
|
||||
GrokTokens {
|
||||
access_token: access_token.into(),
|
||||
refresh_token: Some("refresh".into()),
|
||||
expires_at: expires_in.map(|secs| SystemTime::now() + Duration::from_secs(secs)),
|
||||
connected_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn geap_credentials(access_token: &str, expires_in: Option<u64>) -> GeapCredentials {
|
||||
GeapCredentials::new(
|
||||
access_token.into(),
|
||||
expires_in.map(|secs| SystemTime::now() + Duration::from_secs(secs)),
|
||||
)
|
||||
}
|
||||
|
||||
fn geap_binding() -> GeapMintBinding {
|
||||
GeapMintBinding {
|
||||
user_uid: "user-1".into(),
|
||||
audience:
|
||||
"//iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/p/providers/q"
|
||||
.into(),
|
||||
federation: GeapFederation::ServiceAccount {
|
||||
email: "sa@proj.iam.gserviceaccount.com".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// The expected binding the request build site passes in is the same type as
|
||||
// the stored `minted_for`, so the attach check is a plain `==`.
|
||||
fn geap_gate() -> GeapMintBinding {
|
||||
geap_binding()
|
||||
}
|
||||
|
||||
fn geap_loaded(access_token: &str, expires_in: Option<u64>) -> GeapCredentialsState {
|
||||
GeapCredentialsState::Loaded {
|
||||
credentials: geap_credentials(access_token, expires_in),
|
||||
loaded_at: SystemTime::now(),
|
||||
minted_for: geap_binding(),
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint(
|
||||
name: &str,
|
||||
url: &str,
|
||||
api_key: &str,
|
||||
models: &[(&str, Option<&str>)],
|
||||
) -> CustomEndpoint {
|
||||
endpoint_with_keys(
|
||||
name,
|
||||
url,
|
||||
api_key,
|
||||
&models
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (n, a))| (*n, *a, format!("cfg-{i}")))
|
||||
.collect::<Vec<_>>()
|
||||
.iter()
|
||||
.map(|(n, a, k)| (*n, *a, k.as_str()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
|
||||
fn endpoint_with_keys(
|
||||
name: &str,
|
||||
url: &str,
|
||||
api_key: &str,
|
||||
models: &[(&str, Option<&str>, &str)],
|
||||
) -> CustomEndpoint {
|
||||
CustomEndpoint {
|
||||
name: name.into(),
|
||||
url: url.into(),
|
||||
api_key: api_key.into(),
|
||||
models: models
|
||||
.iter()
|
||||
.map(|(n, a, cfg)| CustomEndpointModel {
|
||||
name: (*n).into(),
|
||||
alias: a.map(|s| s.into()),
|
||||
config_key: (*cfg).into(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ── serde round-trip ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn serde_round_trip_empty() {
|
||||
let keys = ApiKeys::default();
|
||||
let json = serde_json::to_string(&keys).unwrap();
|
||||
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(keys, deser);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_round_trip_with_provider_keys() {
|
||||
let keys = ApiKeys {
|
||||
openai: Some("sk-openai".into()),
|
||||
anthropic: Some("sk-ant-abc".into()),
|
||||
google: Some("AIzaSy123".into()),
|
||||
open_router: Some("sk-or-xxx".into()),
|
||||
custom_endpoints: vec![],
|
||||
};
|
||||
let json = serde_json::to_string(&keys).unwrap();
|
||||
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(keys, deser);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_round_trip_with_custom_endpoints() {
|
||||
let keys = ApiKeys {
|
||||
openai: None,
|
||||
anthropic: None,
|
||||
google: None,
|
||||
open_router: None,
|
||||
custom_endpoints: vec![
|
||||
endpoint("ep1", "https://a.io/v1", "key1", &[("gpt-4", Some("fast"))]),
|
||||
endpoint(
|
||||
"ep2",
|
||||
"https://b.io/v1",
|
||||
"key2",
|
||||
&[("llama-70b", None), ("mixtral", Some("mix"))],
|
||||
),
|
||||
],
|
||||
};
|
||||
let json = serde_json::to_string(&keys).unwrap();
|
||||
let deser: ApiKeys = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(keys, deser);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_ignores_unknown_fields() {
|
||||
let json = r#"{"openai":"sk-x","unknown_field":"value","custom_endpoints":[]}"#;
|
||||
let keys: ApiKeys = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(keys.openai, Some("sk-x".into()));
|
||||
assert!(keys.custom_endpoints.is_empty());
|
||||
}
|
||||
|
||||
// ── has_any_key ─────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn has_any_key_false_when_empty() {
|
||||
assert!(!ApiKeys::default().has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_any_key_true_for_openai_only() {
|
||||
let keys = ApiKeys {
|
||||
openai: Some("sk-x".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(keys.has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_any_key_true_for_custom_endpoints_only() {
|
||||
let keys = ApiKeys {
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "key", &[("m", None)])],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(keys.has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_any_key_false_for_endpoint_with_empty_api_key() {
|
||||
let keys = ApiKeys {
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "", &[("m", None)])],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!keys.has_any_key());
|
||||
}
|
||||
|
||||
// ── provider_key_count ─────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn provider_key_count_zero_when_empty() {
|
||||
assert_eq!(ApiKeys::default().provider_key_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_key_count_counts_each_provider_key() {
|
||||
let keys = ApiKeys {
|
||||
openai: Some("sk-o".into()),
|
||||
anthropic: Some("sk-a".into()),
|
||||
google: Some("AIza".into()),
|
||||
open_router: Some("sk-or".into()),
|
||||
custom_endpoints: vec![],
|
||||
};
|
||||
assert_eq!(keys.provider_key_count(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_key_count_ignores_blank_keys_and_endpoints() {
|
||||
let keys = ApiKeys {
|
||||
openai: Some("sk-o".into()),
|
||||
anthropic: Some(" ".into()),
|
||||
google: None,
|
||||
open_router: None,
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
|
||||
};
|
||||
// Only the non-blank OpenAI key counts; the whitespace Anthropic key and the
|
||||
// custom endpoint are excluded.
|
||||
assert_eq!(keys.provider_key_count(), 1);
|
||||
}
|
||||
|
||||
// ── custom_model_providers_for_request ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn custom_model_providers_none_when_empty() {
|
||||
let mgr = make_manager(ApiKeys::default());
|
||||
assert!(mgr.custom_model_providers_for_request(true).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_model_providers_none_when_byo_disabled() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(mgr.custom_model_providers_for_request(false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_model_providers_populates_single_endpoint() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![endpoint_with_keys(
|
||||
"My EP",
|
||||
"https://custom.io/v1",
|
||||
"ep-key",
|
||||
&[("big-model", Some("alias"), "uuid-1")],
|
||||
)],
|
||||
..Default::default()
|
||||
});
|
||||
let result = mgr.custom_model_providers_for_request(true).unwrap();
|
||||
assert_eq!(result.providers.len(), 1);
|
||||
let p = &result.providers[0];
|
||||
assert_eq!(p.base_url, "https://custom.io/v1");
|
||||
assert_eq!(p.api_key, "ep-key");
|
||||
assert_eq!(p.models.len(), 1);
|
||||
assert_eq!(p.models[0].slug, "big-model");
|
||||
assert_eq!(p.models[0].config_key, "uuid-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_endpoints_all_serialize() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![
|
||||
endpoint_with_keys(
|
||||
"ep1",
|
||||
"https://a.io",
|
||||
"k1",
|
||||
&[("gpt-4", Some("fast"), "uuid-a")],
|
||||
),
|
||||
endpoint_with_keys(
|
||||
"ep2",
|
||||
"https://b.io",
|
||||
"k2",
|
||||
&[
|
||||
("llama-70b", None, "uuid-b"),
|
||||
("mixtral", Some("mix"), "uuid-c"),
|
||||
],
|
||||
),
|
||||
],
|
||||
..Default::default()
|
||||
});
|
||||
let result = mgr.custom_model_providers_for_request(true).unwrap();
|
||||
assert_eq!(result.providers.len(), 2);
|
||||
assert_eq!(result.providers[0].base_url, "https://a.io");
|
||||
assert_eq!(result.providers[0].models[0].config_key, "uuid-a");
|
||||
assert_eq!(result.providers[1].base_url, "https://b.io");
|
||||
assert_eq!(result.providers[1].models.len(), 2);
|
||||
assert_eq!(result.providers[1].models[0].slug, "llama-70b");
|
||||
assert_eq!(result.providers[1].models[0].config_key, "uuid-b");
|
||||
assert_eq!(result.providers[1].models[1].config_key, "uuid-c");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byok_disabled_returns_none_even_with_endpoints() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(mgr.custom_model_providers_for_request(false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_api_key_endpoints_are_skipped() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![
|
||||
endpoint_with_keys("empty", "https://a.io", "", &[("m", None, "uuid-x")]),
|
||||
endpoint_with_keys("ok", "https://b.io", "k", &[("m", None, "uuid-y")]),
|
||||
],
|
||||
..Default::default()
|
||||
});
|
||||
let result = mgr.custom_model_providers_for_request(true).unwrap();
|
||||
assert_eq!(result.providers.len(), 1);
|
||||
assert_eq!(result.providers[0].base_url, "https://b.io");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoints_with_only_empty_models_are_skipped() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![endpoint_with_keys(
|
||||
"ep",
|
||||
"https://a.io",
|
||||
"k",
|
||||
&[("", None, "uuid-z")],
|
||||
)],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(mgr.custom_model_providers_for_request(true).is_none());
|
||||
}
|
||||
|
||||
// ── display_label fallback ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn display_label_uses_alias_when_present() {
|
||||
let m = CustomEndpointModel {
|
||||
name: "raw-name".into(),
|
||||
alias: Some("My Alias".into()),
|
||||
config_key: "k".into(),
|
||||
};
|
||||
assert_eq!(m.display_label(), "My Alias");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_label_falls_back_to_name_when_alias_missing() {
|
||||
let m = CustomEndpointModel {
|
||||
name: "raw-name".into(),
|
||||
alias: None,
|
||||
config_key: "k".into(),
|
||||
};
|
||||
assert_eq!(m.display_label(), "raw-name");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_label_falls_back_to_name_when_alias_is_whitespace() {
|
||||
let m = CustomEndpointModel {
|
||||
name: "raw-name".into(),
|
||||
alias: Some(" ".into()),
|
||||
config_key: "k".into(),
|
||||
};
|
||||
assert_eq!(m.display_label(), "raw-name");
|
||||
}
|
||||
|
||||
// ── api_keys_for_request ────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_none_when_empty() {
|
||||
let mgr = make_manager(ApiKeys::default());
|
||||
assert!(mgr.api_keys_for_request(true, false, None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_populates_provider_keys() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
openai: Some("sk-o".into()),
|
||||
anthropic: Some("sk-a".into()),
|
||||
..Default::default()
|
||||
});
|
||||
let result = mgr.api_keys_for_request(true, false, None).unwrap();
|
||||
assert_eq!(result.openai, "sk-o");
|
||||
assert_eq!(result.anthropic, "sk-a");
|
||||
assert!(result.google.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_keys_when_byo_disabled() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
openai: Some("sk-o".into()),
|
||||
..Default::default()
|
||||
});
|
||||
// With BYO disabled and no other credentials, returns None.
|
||||
assert!(mgr.api_keys_for_request(false, false, None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_none_for_custom_endpoints_only() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
custom_endpoints: vec![endpoint("ep", "https://a.io", "k", &[("m", None)])],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(mgr.api_keys_for_request(true, false, None).is_none());
|
||||
}
|
||||
|
||||
// ── grok oauth token ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn grok_access_token_present_without_expiry() {
|
||||
let t = GrokTokens {
|
||||
access_token: "tok".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(t.access_token_for_request(), Some("tok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_access_token_blank_is_none() {
|
||||
let t = GrokTokens {
|
||||
access_token: " ".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(t.access_token_for_request(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_access_token_near_expiry_still_sent() {
|
||||
// Expired tokens are still sent; the server is the authority on validity.
|
||||
let t = grok_tokens("tok", Some(0));
|
||||
assert_eq!(t.access_token_for_request(), Some("tok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_access_token_far_future_is_some() {
|
||||
let t = grok_tokens("tok", Some(3600));
|
||||
assert_eq!(t.access_token_for_request(), Some("tok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_needs_refresh_within_lead_time() {
|
||||
assert!(grok_tokens("tok", Some(30)).needs_refresh(Duration::from_secs(300)));
|
||||
assert!(!grok_tokens("tok", Some(3600)).needs_refresh(Duration::from_secs(300)));
|
||||
// Expired tokens still need a refresh.
|
||||
assert!(grok_tokens("tok", Some(0)).needs_refresh(Duration::from_secs(300)));
|
||||
// Unknown expiry never reports as needing refresh.
|
||||
assert!(!grok_tokens("tok", None).needs_refresh(Duration::from_secs(300)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_includes_grok_token() {
|
||||
let mgr = make_manager_with_grok(
|
||||
ApiKeys::default(),
|
||||
Some(grok_tokens("grok-abc", Some(3600))),
|
||||
);
|
||||
let result = mgr.api_keys_for_request(true, false, None).unwrap();
|
||||
assert_eq!(result.grok_oauth_access_token, "grok-abc");
|
||||
assert!(result.anthropic.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_grok_token_when_byo_disabled() {
|
||||
// The Grok subscription is user-provided auth, so it follows the BYO
|
||||
// policy gate: with BYO disabled and no other credentials, returns None.
|
||||
let mgr = make_manager_with_grok(
|
||||
ApiKeys::default(),
|
||||
Some(grok_tokens("grok-abc", Some(3600))),
|
||||
);
|
||||
assert!(mgr.api_keys_for_request(false, false, None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_includes_expired_grok_token() {
|
||||
// Expired tokens are still sent in requests; the server rejects truly
|
||||
// invalid ones and the background refresh replaces them.
|
||||
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens("grok-abc", Some(0))));
|
||||
let result = mgr.api_keys_for_request(true, false, None).unwrap();
|
||||
assert_eq!(result.grok_oauth_access_token, "grok-abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_grok_subscription_false_when_not_connected() {
|
||||
let mgr = make_manager(ApiKeys::default());
|
||||
assert!(!mgr.has_grok_subscription());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_grok_subscription_true_when_connected() {
|
||||
let mgr = make_manager_with_grok(
|
||||
ApiKeys::default(),
|
||||
Some(grok_tokens("grok-abc", Some(3600))),
|
||||
);
|
||||
assert!(mgr.has_grok_subscription());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_grok_subscription_true_for_expired_token() {
|
||||
// A connected subscription still counts even when its token is past expiry:
|
||||
// the token is sent anyway and the server is the authority on validity.
|
||||
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens("grok-abc", Some(0))));
|
||||
assert!(mgr.has_grok_subscription());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_grok_subscription_false_when_token_blank() {
|
||||
// A blank token can't be sent, so it does not count as a usable credential.
|
||||
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens(" ", None)));
|
||||
assert!(!mgr.has_grok_subscription());
|
||||
}
|
||||
|
||||
// ── ApiKeyManager::has_any_key ──────────────────
|
||||
|
||||
#[test]
|
||||
fn manager_has_any_key_false_when_no_keys_and_no_grok() {
|
||||
let mgr = make_manager(ApiKeys::default());
|
||||
assert!(!mgr.has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_has_any_key_true_for_pasted_key_without_grok() {
|
||||
let mgr = make_manager(ApiKeys {
|
||||
openai: Some("sk-x".into()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(mgr.has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_has_any_key_true_for_connected_grok_without_pasted_key() {
|
||||
// The crux: a connected Grok subscription counts even with no pasted keys,
|
||||
// matching how it's sent as a BYO credential on requests.
|
||||
let mgr = make_manager_with_grok(
|
||||
ApiKeys::default(),
|
||||
Some(grok_tokens("grok-abc", Some(3600))),
|
||||
);
|
||||
assert!(mgr.has_any_key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_has_any_key_false_for_blank_grok_and_no_keys() {
|
||||
let mgr = make_manager_with_grok(ApiKeys::default(), Some(grok_tokens(" ", None)));
|
||||
assert!(!mgr.has_any_key());
|
||||
}
|
||||
|
||||
// ── geap credentials ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn geap_access_token_present_without_expiry() {
|
||||
let credentials = GeapCredentials::new("tok".into(), None);
|
||||
assert_eq!(credentials.access_token_for_request(), Some("tok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn geap_access_token_blank_is_none() {
|
||||
let credentials = GeapCredentials::new(" ".into(), None);
|
||||
assert_eq!(credentials.access_token_for_request(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn geap_access_token_near_expiry_still_sent() {
|
||||
// Expired tokens are still sent; Google is the authority on validity.
|
||||
let credentials = geap_credentials("tok", Some(0));
|
||||
assert_eq!(credentials.access_token_for_request(), Some("tok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn geap_needs_refresh_lead_time_boundaries() {
|
||||
// Within the 5-minute lead window.
|
||||
assert!(geap_credentials("tok", Some(30)).needs_refresh());
|
||||
// Comfortably fresh.
|
||||
assert!(!geap_credentials("tok", Some(3600)).needs_refresh());
|
||||
// Already expired -> still needs a refresh.
|
||||
assert!(geap_credentials("tok", Some(0)).needs_refresh());
|
||||
// Unknown expiry never reports as needing a refresh.
|
||||
assert!(!geap_credentials("tok", None).needs_refresh());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_includes_geap_token_when_gate_and_binding_match() {
|
||||
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
|
||||
let result = mgr
|
||||
.api_keys_for_request(false, false, Some(geap_gate()))
|
||||
.unwrap();
|
||||
let credentials = result.google_cloud_credentials.unwrap();
|
||||
assert_eq!(credentials.access_token, "geap-abc");
|
||||
// The GEAP token is independent of the BYO key gate.
|
||||
assert!(result.anthropic.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_includes_expired_geap_token() {
|
||||
// Expired tokens are still attached — never silently dropped. Google
|
||||
// rejects truly invalid ones, which surfaces a recoverable error instead
|
||||
// of a silent fallback to another route.
|
||||
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(0)));
|
||||
let result = mgr
|
||||
.api_keys_for_request(false, false, Some(geap_gate()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.google_cloud_credentials.unwrap().access_token,
|
||||
"geap-abc"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_geap_token_without_gate() {
|
||||
// No gate (policy off at the call site) ⇒ no GEAP credentials, even when
|
||||
// a token is loaded.
|
||||
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
|
||||
assert!(mgr.api_keys_for_request(false, false, None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_geap_token_on_binding_mismatch() {
|
||||
let mgr = make_manager_with_geap(geap_loaded("geap-abc", Some(3600)));
|
||||
|
||||
// A different user (sign-out/account switch).
|
||||
let mut gate = geap_gate();
|
||||
gate.user_uid = "someone-else".into();
|
||||
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
|
||||
|
||||
// A different audience (admin changed the pool/provider).
|
||||
let mut gate = geap_gate();
|
||||
gate.audience = "//iam.googleapis.com/projects/2/locations/global/workloadIdentityPools/other/providers/other".into();
|
||||
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
|
||||
|
||||
// A different service account (admin changed impersonation target).
|
||||
let mut gate = geap_gate();
|
||||
gate.federation = GeapFederation::ServiceAccount {
|
||||
email: "other@proj.iam.gserviceaccount.com".into(),
|
||||
};
|
||||
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_serves_previous_geap_token_while_refreshing() {
|
||||
// A re-mint in flight keeps serving the previous token — tokens stay
|
||||
// until replaced.
|
||||
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing {
|
||||
previous: Some((geap_credentials("geap-old", Some(10)), geap_binding())),
|
||||
});
|
||||
let result = mgr
|
||||
.api_keys_for_request(false, false, Some(geap_gate()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.google_cloud_credentials.unwrap().access_token,
|
||||
"geap-old"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_geap_token_during_first_mint() {
|
||||
// The very first mint has nothing to serve yet.
|
||||
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing { previous: None });
|
||||
assert!(mgr
|
||||
.api_keys_for_request(false, false, Some(geap_gate()))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_geap_token_for_non_loaded_states() {
|
||||
for state in [
|
||||
GeapCredentialsState::Missing,
|
||||
GeapCredentialsState::Disabled,
|
||||
GeapCredentialsState::Failed {
|
||||
error: LoadGeapCredentialsError::ExchangeToken {
|
||||
status: None,
|
||||
detail: "boom".into(),
|
||||
},
|
||||
},
|
||||
] {
|
||||
let mgr = make_manager_with_geap(state);
|
||||
assert!(mgr
|
||||
.api_keys_for_request(false, false, Some(geap_gate()))
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_keys_for_request_omits_geap_token_when_previous_binding_mismatches() {
|
||||
let mgr = make_manager_with_geap(GeapCredentialsState::Refreshing {
|
||||
previous: Some((geap_credentials("geap-old", Some(10)), geap_binding())),
|
||||
});
|
||||
let mut gate = geap_gate();
|
||||
gate.user_uid = "someone-else".into();
|
||||
assert!(mgr.api_keys_for_request(false, false, Some(gate)).is_none());
|
||||
}
|
||||
@@ -6,7 +6,7 @@ use warp_multi_agent_api as api;
|
||||
|
||||
/// Temporary AWS credentials loaded from the AWS SDK.
|
||||
/// These are not persisted and are only used at runtime.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct AwsCredentials {
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
@@ -14,6 +14,20 @@ pub struct AwsCredentials {
|
||||
expires_at: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AwsCredentials {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AwsCredentials")
|
||||
.field("access_key", &"<redacted>")
|
||||
.field("secret_key", &"<redacted>")
|
||||
.field(
|
||||
"session_token",
|
||||
&self.session_token.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field("expires_at", &self.expires_at)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl AwsCredentials {
|
||||
pub fn new(
|
||||
access_key: String,
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
use std::cmp::Ordering;
|
||||
use std::fmt::{self, Display};
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use itertools::{EitherOrBoth, Itertools};
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt::{self, Display},
|
||||
ops::Range,
|
||||
path::PathBuf,
|
||||
sync::LazyLock,
|
||||
};
|
||||
use strsim::jaro_winkler;
|
||||
lazy_static! {
|
||||
/// Regex to parse a line number from a string in the format "{number}|{line}"
|
||||
@@ -283,6 +282,35 @@ fn unmatched_line_suffix<'a>(search_line: &str, file_line: &'a str) -> Option<&'
|
||||
}
|
||||
}
|
||||
|
||||
/// Preserve the unmatched suffix of the final matched file line when the LLM emitted only a
|
||||
/// partial final line.
|
||||
///
|
||||
/// When search and replace have the same line count, preserve the suffix to maintain the legacy
|
||||
/// behavior for simple partial-line replacements. When line counts differ, only preserve it if the
|
||||
/// replacement's final line still carries the same partial context as the search's final line.
|
||||
fn append_unmatched_line_suffix(search: &str, file_line: &str, insertion: &mut String) {
|
||||
let Some(search_last_line) = lines(search).last() else {
|
||||
return;
|
||||
};
|
||||
let Some(suffix) = unmatched_line_suffix(search_last_line, file_line) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let search_line_count = lines(search).count();
|
||||
let insertion_line_count = lines(insertion).count();
|
||||
let Some(insertion_last_line) = lines(insertion).last() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if search_line_count != insertion_line_count
|
||||
&& search_last_line.trim_start() != insertion_last_line.trim_start()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let insertion_point = insertion.trim_end_matches('\n').len();
|
||||
insertion.insert_str(insertion_point, suffix);
|
||||
}
|
||||
/// We told the model not to include line numbers for the replacement content. However, it can
|
||||
/// still happen. Try to remove them here.
|
||||
/// https://github.com/warpdotdev/warp-server/blob/d9c1b6d1443290f2355979ae552d41af01a63bde/logic/ai/prompt/tools/suggest_diff.yaml#L34-L34
|
||||
@@ -402,6 +430,14 @@ pub fn fuzzy_match_v4a_diffs(
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by start line and remove overlapping deltas. When the LLM produces
|
||||
// multiple hunks targeting the same region (e.g. a large deletion whose
|
||||
// matched range subsumes a nearby single-line edit), the overlapping delta
|
||||
// must be dropped — applying both would produce an invalid edit range in
|
||||
// the editor buffer (see WARP-CLIENT-DEV-NYY).
|
||||
deltas.sort_by_key(|d| d.replacement_line_range.start);
|
||||
deltas = deduplicate_overlapping_deltas(deltas);
|
||||
|
||||
let update_deltas_empty = deltas.is_empty();
|
||||
let failures = if failures.fuzzy_match_failures > 0
|
||||
|| failures.missing_line_numbers > 0
|
||||
@@ -420,6 +456,33 @@ pub fn fuzzy_match_v4a_diffs(
|
||||
}
|
||||
}
|
||||
|
||||
/// Given a list of `DiffDelta`s sorted by `replacement_line_range.start`,
|
||||
/// drop any delta whose range overlaps with the preceding accepted delta.
|
||||
///
|
||||
/// "Overlaps" means `B.start < A.end` (strictly inside or partial overlap).
|
||||
/// Adjacent ranges (`A.end == B.start`) are kept.
|
||||
fn deduplicate_overlapping_deltas(sorted_deltas: Vec<DiffDelta>) -> Vec<DiffDelta> {
|
||||
let mut result: Vec<DiffDelta> = Vec::with_capacity(sorted_deltas.len());
|
||||
|
||||
for delta in sorted_deltas {
|
||||
let dominated = result.last().is_some_and(|prev| {
|
||||
delta.replacement_line_range.start < prev.replacement_line_range.end
|
||||
});
|
||||
if dominated {
|
||||
log::warn!(
|
||||
"Dropping V4A delta with overlapping range {:?} \
|
||||
(subsumed by preceding delta with range {:?})",
|
||||
delta.replacement_line_range,
|
||||
result.last().unwrap().replacement_line_range,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
result.push(delta);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn fuzzy_match_file_diffs(
|
||||
diffs: &[SearchAndReplace],
|
||||
file_content: &str,
|
||||
@@ -547,13 +610,12 @@ fn fuzzy_match_file_diffs(
|
||||
// the delta would replace the entire line and drop the unmatched
|
||||
// suffix. Detect this and preserve the suffix in the insertion.
|
||||
let mut insertion = diff.replace.clone();
|
||||
if range.end >= 2 && lines(&search).count() == lines(&insertion).count() {
|
||||
if let Some(suffix) = lines(&search)
|
||||
.last()
|
||||
.and_then(|last| unmatched_line_suffix(last, target_lines[range.end - 2]))
|
||||
{
|
||||
insertion.push_str(suffix);
|
||||
}
|
||||
if range.end >= 2 {
|
||||
append_unmatched_line_suffix(
|
||||
&search,
|
||||
target_lines[range.end - 2],
|
||||
&mut insertion,
|
||||
);
|
||||
}
|
||||
deltas.push(DiffDelta {
|
||||
replacement_line_range: range.start..range.end,
|
||||
@@ -1208,5 +1270,5 @@ fn find_change_context_start(change_context: &[String], file_lines: &[&str]) ->
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "mod_test.rs"]
|
||||
#[path = "mod_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
+145
@@ -608,6 +608,66 @@ fn test_partial_last_line_in_search_preserves_suffix() {
|
||||
assert_eq!(result, "func foo() {\nlet y = 1;\nlet x = 2;\n}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_last_line_in_multiline_replacement_preserves_suffix() {
|
||||
// This mirrors a model edit that deletes middle lines while leaving the final line as partial
|
||||
// trailing context. The final line should remain a no-op after suffix preservation.
|
||||
let file_content = "\
|
||||
mod proxy;
|
||||
pub fn run_daemon() -> anyhow::Result<()> {
|
||||
// Logging is now handled by init_common (log_destination: File).
|
||||
|
||||
// socket_path: ~/.warp[-channel]/remote-server/server.sock
|
||||
// The Unix domain socket the daemon binds on.
|
||||
}
|
||||
";
|
||||
|
||||
let diffs = [SearchAndReplace {
|
||||
search: "\
|
||||
2|pub fn run_daemon() -> anyhow::Result<()> {
|
||||
3| // Logging is now handled by init_common (log_destination: File).
|
||||
4|
|
||||
5| // socket_path:"
|
||||
.to_string(),
|
||||
replace: "\
|
||||
pub fn run_daemon() -> anyhow::Result<()> {
|
||||
// socket_path:"
|
||||
.to_string(),
|
||||
}];
|
||||
|
||||
let (deltas, _failures) = fuzzy_match_file_diffs(&diffs, file_content);
|
||||
|
||||
assert_eq!(deltas.len(), 1, "Expected one matched delta");
|
||||
assert_eq!(deltas[0].replacement_line_range, 2..6);
|
||||
assert_eq!(
|
||||
deltas[0].insertion,
|
||||
"pub fn run_daemon() -> anyhow::Result<()> {\n // socket_path: ~/.warp[-channel]/remote-server/server.sock"
|
||||
);
|
||||
|
||||
let file_lines: Vec<&str> = file_content.lines().collect();
|
||||
let range = &deltas[0].replacement_line_range;
|
||||
let mut result = String::new();
|
||||
for line in &file_lines[..range.start - 1] {
|
||||
result.push_str(line);
|
||||
result.push('\n');
|
||||
}
|
||||
result.push_str(&deltas[0].insertion);
|
||||
result.push('\n');
|
||||
for line in &file_lines[range.end - 1..] {
|
||||
result.push_str(line);
|
||||
result.push('\n');
|
||||
}
|
||||
assert_eq!(
|
||||
result,
|
||||
"\
|
||||
mod proxy;
|
||||
pub fn run_daemon() -> anyhow::Result<()> {
|
||||
// socket_path: ~/.warp[-channel]/remote-server/server.sock
|
||||
// The Unix domain socket the daemon binds on.
|
||||
}
|
||||
"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn test_search_and_replace_accommodates_none() {
|
||||
let parsed_diff = ParsedDiff::StrReplaceEdit {
|
||||
@@ -679,3 +739,88 @@ fn test_custom_lines() {
|
||||
assert_eq!(lines("foo\nbar").collect_vec(), vec!["foo", "bar"]);
|
||||
assert_eq!(lines("foo\nbar\n").collect_vec(), vec!["foo", "bar"]);
|
||||
}
|
||||
|
||||
/// Regression test for WARP-CLIENT-DEV-NYY: panic "Invalid edit range 4042..3982".
|
||||
///
|
||||
/// Reproduces the crash from MAA conversation d71bf84b (request b621adb3).
|
||||
/// Two V4A hunks target the same region: a large deletion whose matched range
|
||||
/// subsumes a nearby single-line edit. Without `deduplicate_overlapping_deltas`,
|
||||
/// both deltas survive and `Buffer::edit` panics on the overlapping ranges.
|
||||
#[test]
|
||||
fn test_v4a_maa_crash_d71bf84b_no_overlapping_deltas() {
|
||||
// File content where hunk A (deletion) and hunk B (delegate tweak) both
|
||||
// match, and hunk A's matched range fully contains hunk B's.
|
||||
// The `ActiveMicButtonTheme.background` line that hunk B targets sits
|
||||
// inside `DefaultWeightAgentInputButtonTheme`'s impl, so hunk A's
|
||||
// deletion (which covers the whole impl) subsumes hunk B.
|
||||
let file_content = "\
|
||||
}\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
struct DefaultWeightAgentInputButtonTheme;\n\
|
||||
\n\
|
||||
impl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n\
|
||||
fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {\n\
|
||||
AgentInputButtonTheme.background(hovered, appearance)\n\
|
||||
}\n\
|
||||
\n\
|
||||
fn text_color(\n\
|
||||
&self,\n\
|
||||
hovered: bool,\n\
|
||||
background: Option<Fill>,\n\
|
||||
appearance: &Appearance,\n\
|
||||
) -> ColorU {\n\
|
||||
AgentInputButtonTheme.text_color(hovered, background, appearance)\n\
|
||||
}\n\
|
||||
\n\
|
||||
fn border(&self, appearance: &Appearance) -> Option<ColorU> {\n\
|
||||
AgentInputButtonTheme.border(appearance)\n\
|
||||
}\n\
|
||||
\n\
|
||||
fn should_opt_out_of_contrast_adjustment(&self) -> bool {\n\
|
||||
true\n\
|
||||
}\n\
|
||||
}";
|
||||
|
||||
let hunks = vec![
|
||||
// Hunk A: delete the entire DefaultWeightAgentInputButtonTheme block.
|
||||
V4AHunk {
|
||||
change_context: vec![],
|
||||
pre_context: " }\n }\n}".to_string(),
|
||||
old: "\nstruct DefaultWeightAgentInputButtonTheme;\n\nimpl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {\n AgentInputButtonTheme.background(hovered, appearance)\n }\n\n fn text_color(\n &self,\n hovered: bool,\n background: Option<Fill>,\n appearance: &Appearance,\n ) -> ColorU {\n AgentInputButtonTheme.text_color(hovered, background, appearance)\n }\n\n fn border(&self, appearance: &Appearance) -> Option<ColorU> {\n AgentInputButtonTheme.border(appearance)\n }\n\n fn should_opt_out_of_contrast_adjustment(&self) -> bool {\n true\n }\n}".to_string(),
|
||||
new: String::new(),
|
||||
post_context: String::new(),
|
||||
},
|
||||
// Hunk B: tweak a delegate call inside the same region hunk A deletes.
|
||||
// Its preContext + old match a line inside hunk A's range, so it
|
||||
// produces a delta whose range overlaps with hunk A's.
|
||||
V4AHunk {
|
||||
change_context: vec![],
|
||||
pre_context: "impl ActionButtonTheme for DefaultWeightAgentInputButtonTheme {\n fn background(&self, hovered: bool, appearance: &Appearance) -> Option<Fill> {".to_string(),
|
||||
old: " AgentInputButtonTheme.background(hovered, appearance)".to_string(),
|
||||
new: " AgentInputButtonTheme::default().background(hovered, appearance)".to_string(),
|
||||
post_context: " }".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let diff = fuzzy_match_v4a_diffs("mod.rs", &hunks, None, file_content);
|
||||
let deltas = deltas(&diff);
|
||||
|
||||
// Hunk B's matched range is inside hunk A's, so deduplication must drop it.
|
||||
// Only hunk A's delta (the deletion) should survive.
|
||||
assert_eq!(
|
||||
deltas.len(),
|
||||
1,
|
||||
"Expected 1 delta (subsumed hunk should be dropped), got {}: {:?}",
|
||||
deltas.len(),
|
||||
deltas
|
||||
.iter()
|
||||
.map(|d| &d.replacement_line_range)
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
assert!(
|
||||
deltas[0].insertion.is_empty(),
|
||||
"The surviving delta should be the deletion"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use warp_multi_agent_api as api;
|
||||
|
||||
/// Refresh the access token this long before its hard expiry
|
||||
pub const GEAP_REFRESH_LEAD_TIME: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct GeapCredentials {
|
||||
access_token: String,
|
||||
expires_at: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GeapCredentials {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("GeapCredentials")
|
||||
.field("access_token", &"<redacted>")
|
||||
.field("expires_at", &self.expires_at)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl GeapCredentials {
|
||||
pub fn new(access_token: String, expires_at: Option<SystemTime>) -> Self {
|
||||
Self {
|
||||
access_token,
|
||||
expires_at,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn expires_at(&self) -> Option<SystemTime> {
|
||||
self.expires_at
|
||||
}
|
||||
|
||||
pub fn access_token_for_request(&self) -> Option<&str> {
|
||||
(!self.access_token.trim().is_empty()).then_some(self.access_token.as_str())
|
||||
}
|
||||
|
||||
pub fn needs_refresh(&self) -> bool {
|
||||
match self.expires_at {
|
||||
Some(expires_at) => expires_at <= SystemTime::now() + GEAP_REFRESH_LEAD_TIME,
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GeapCredentials> for api::request::settings::api_keys::GoogleCloudCredentials {
|
||||
fn from(credentials: GeapCredentials) -> Self {
|
||||
Self {
|
||||
access_token: credentials.access_token,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum GeapFederation {
|
||||
DirectWif,
|
||||
ServiceAccount { email: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct GeapMintBinding {
|
||||
pub user_uid: String,
|
||||
pub audience: String,
|
||||
pub federation: GeapFederation,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum LoadGeapCredentialsError {
|
||||
MintIdentityToken { detail: String },
|
||||
ExchangeToken { status: Option<u16>, detail: String },
|
||||
ImpersonateServiceAccount { status: Option<u16>, detail: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub enum GeapCredentialsState {
|
||||
#[default]
|
||||
Missing,
|
||||
Disabled,
|
||||
Refreshing {
|
||||
previous: Option<(GeapCredentials, GeapMintBinding)>,
|
||||
},
|
||||
Loaded {
|
||||
credentials: GeapCredentials,
|
||||
loaded_at: SystemTime,
|
||||
minted_for: GeapMintBinding,
|
||||
},
|
||||
Failed {
|
||||
error: LoadGeapCredentialsError,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Refresh orchestration for a connected xAI / Grok subscription's OAuth
|
||||
//! tokens.
|
||||
//!
|
||||
//! The tokens themselves live in [`ApiKeyManager`] (the request-building
|
||||
//! source of truth, persisted to secure storage under `GrokOAuthTokens`).
|
||||
//! This module owns the network-facing refresh lifecycle — converting a
|
||||
//! [`TokenResponse`] into stored [`GrokTokens`], proactively refreshing the
|
||||
//! access token shortly before it expires, and rescheduling the next refresh.
|
||||
//!
|
||||
//! The Grok subscription is BYO auth, so background refresh follows the BYO
|
||||
//! API key policy. That policy lives in the app layer (workspace settings),
|
||||
//! which this crate has no visibility into; the app wires it in via
|
||||
//! [`ApiKeyManager::set_grok_refresh_allowed`].
|
||||
//!
|
||||
//! The network/protocol side of the connect flow (authorize URL, loopback
|
||||
//! callback server, token exchange/refresh) lives in the [`oauth`] submodule.
|
||||
|
||||
pub mod oauth;
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use galaxyui_core::r#async::Timer;
|
||||
use galaxyui_core::ModelContext;
|
||||
|
||||
use self::oauth::TokenResponse;
|
||||
use crate::api_keys::{ApiKeyManager, GrokTokens};
|
||||
|
||||
/// Refresh the access token this long before its hard expiry so a request
|
||||
/// never races the expiration. Possibly-expired tokens are still sent (the
|
||||
/// server is the authority on validity), so this lead time is purely about
|
||||
/// keeping the token fresh, not about when it stops being sent.
|
||||
const REFRESH_LEAD_TIME: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Builds [`GrokTokens`] from a token-endpoint [`TokenResponse`], computing the
|
||||
/// absolute `expires_at` from the relative `expires_in`. Values not present in
|
||||
/// the response are carried over from `previous`: the refresh token when xAI
|
||||
/// doesn't return a new one (refresh-token rotation is optional in OAuth 2.0),
|
||||
/// and `connected_at` so it keeps reflecting the initial connection time
|
||||
/// (initialized to now when there are no previous tokens, i.e. a fresh
|
||||
/// connect).
|
||||
pub fn grok_tokens_from_response(
|
||||
response: TokenResponse,
|
||||
previous: Option<&GrokTokens>,
|
||||
) -> GrokTokens {
|
||||
let expires_at = response
|
||||
.expires_in
|
||||
.and_then(|secs| u64::try_from(secs).ok())
|
||||
.and_then(|secs| SystemTime::now().checked_add(Duration::from_secs(secs)));
|
||||
GrokTokens {
|
||||
access_token: response.access_token,
|
||||
refresh_token: response
|
||||
.refresh_token
|
||||
.or_else(|| previous.and_then(|tokens| tokens.refresh_token.clone())),
|
||||
expires_at,
|
||||
connected_at: previous
|
||||
.and_then(|tokens| tokens.connected_at)
|
||||
.or_else(|| Some(SystemTime::now())),
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiKeyManager {
|
||||
/// Persists freshly obtained tokens (e.g. right after the connect flow) and
|
||||
/// schedules the next proactive refresh.
|
||||
pub fn store_grok_tokens(&mut self, response: TokenResponse, ctx: &mut ModelContext<Self>) {
|
||||
apply_grok_tokens(self, response, ctx);
|
||||
}
|
||||
|
||||
/// Updates whether background refresh of the stored Grok tokens is
|
||||
/// allowed. The Grok subscription is BYO auth, so refresh follows the same
|
||||
/// policy gate as request injection ([`Self::api_keys_for_request`]):
|
||||
/// tokens that can never be sent shouldn't be kept fresh. The policy lives
|
||||
/// in the app layer, which calls this at startup and whenever the policy
|
||||
/// may have changed (e.g. team data arriving, or a workspace switch).
|
||||
///
|
||||
/// Schedules a refresh on a disabled -> enabled transition (refreshing
|
||||
/// immediately if the token has already (nearly) expired); in-flight
|
||||
/// timers re-check the flag when they fire. Repeated calls with an
|
||||
/// unchanged value are no-ops, so duplicate timers can't pile up.
|
||||
pub fn set_grok_refresh_allowed(&mut self, allowed: bool, ctx: &mut ModelContext<Self>) {
|
||||
if self.grok_refresh_allowed == allowed {
|
||||
return;
|
||||
}
|
||||
self.grok_refresh_allowed = allowed;
|
||||
if allowed {
|
||||
schedule_grok_token_refresh(self, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/// Request-time safety net: kicks off a background refresh of the stored
|
||||
/// Grok tokens when they are nearing (or already past) expiry, so
|
||||
/// upcoming requests can authenticate even if the proactive refresh loop
|
||||
/// never armed or died (e.g. a stale BYO policy at startup, or an earlier
|
||||
/// failed refresh). The triggering request still carries the currently
|
||||
/// stored token — the server is the authority on its validity.
|
||||
///
|
||||
/// `byo_allowed` is the BYO API key policy as freshly evaluated by the
|
||||
/// caller at request time. It also re-syncs the stored policy mirror,
|
||||
/// which can go stale between `TeamsChanged` events; a disabled ->
|
||||
/// enabled transition re-arms the proactive refresh loop.
|
||||
pub fn refresh_grok_tokens_if_needed(
|
||||
&mut self,
|
||||
byo_allowed: bool,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
self.set_grok_refresh_allowed(byo_allowed, ctx);
|
||||
if !byo_allowed || self.grok_refresh_in_flight {
|
||||
return;
|
||||
}
|
||||
let Some(tokens) = self.grok_tokens() else {
|
||||
return;
|
||||
};
|
||||
if !tokens.needs_refresh(REFRESH_LEAD_TIME) {
|
||||
return;
|
||||
}
|
||||
let Some(refresh_token) = tokens.refresh_token.clone() else {
|
||||
return;
|
||||
};
|
||||
log::info!(
|
||||
"Grok OAuth token is nearing or past expiry at request time; refreshing in background"
|
||||
);
|
||||
spawn_grok_refresh(self, refresh_token, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores the tokens from `response` (carrying over the previous refresh token
|
||||
/// and connection time when absent) and schedules the next proactive refresh.
|
||||
fn apply_grok_tokens(
|
||||
manager: &mut ApiKeyManager,
|
||||
response: TokenResponse,
|
||||
ctx: &mut ModelContext<ApiKeyManager>,
|
||||
) {
|
||||
let tokens = grok_tokens_from_response(response, manager.grok_tokens());
|
||||
manager.set_grok_tokens(Some(tokens), ctx);
|
||||
schedule_grok_token_refresh(manager, ctx);
|
||||
}
|
||||
|
||||
/// Schedules a one-shot proactive refresh [`REFRESH_LEAD_TIME`] before the
|
||||
/// current token's expiry (immediately if already within that window).
|
||||
///
|
||||
/// No-op when there's nothing to refresh against (no tokens, no refresh token,
|
||||
/// or no known expiry). Reschedules itself after each successful refresh, so a
|
||||
/// single call establishes an ongoing refresh loop for the lifetime of the
|
||||
/// connection.
|
||||
fn schedule_grok_token_refresh(manager: &mut ApiKeyManager, ctx: &mut ModelContext<ApiKeyManager>) {
|
||||
// When the BYO API key policy is disabled the token is never sent, so
|
||||
// don't refresh it in the background either. `set_grok_refresh_allowed`
|
||||
// re-establishes the loop if the policy is later enabled.
|
||||
if !manager.grok_refresh_allowed {
|
||||
return;
|
||||
}
|
||||
let Some(tokens) = manager.grok_tokens() else {
|
||||
return;
|
||||
};
|
||||
let Some(refresh_token) = tokens.refresh_token.clone() else {
|
||||
return;
|
||||
};
|
||||
let Some(expires_at) = tokens.expires_at else {
|
||||
// No expiry signal, so there's nothing to schedule against.
|
||||
return;
|
||||
};
|
||||
|
||||
let now = SystemTime::now();
|
||||
let fire_at = expires_at.checked_sub(REFRESH_LEAD_TIME).unwrap_or(now);
|
||||
let delay = fire_at.duration_since(now).unwrap_or(Duration::ZERO);
|
||||
|
||||
ctx.spawn(
|
||||
async move {
|
||||
Timer::after(delay).await;
|
||||
},
|
||||
move |manager, _output, ctx| {
|
||||
// The BYO policy may have flipped off while we slept;
|
||||
// `set_grok_refresh_allowed` restarts the loop if it flips back
|
||||
// on.
|
||||
if !manager.grok_refresh_allowed {
|
||||
return;
|
||||
}
|
||||
// The stored token may have changed (reconnect/disconnect) while we
|
||||
// slept; only refresh if our refresh token is still the current one.
|
||||
let still_current = manager
|
||||
.grok_tokens()
|
||||
.and_then(|t| t.refresh_token.as_deref())
|
||||
== Some(refresh_token.as_str());
|
||||
if still_current {
|
||||
spawn_grok_refresh(manager, refresh_token, ctx);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Kicks off a background token refresh using `refresh_token`, applying the
|
||||
/// result (which reschedules the next refresh) or logging the failure.
|
||||
///
|
||||
/// No-op when a refresh is already in flight, so the proactive timer and the
|
||||
/// request-time safety net can't issue overlapping refreshes.
|
||||
fn spawn_grok_refresh(
|
||||
manager: &mut ApiKeyManager,
|
||||
refresh_token: String,
|
||||
ctx: &mut ModelContext<ApiKeyManager>,
|
||||
) {
|
||||
if manager.grok_refresh_in_flight {
|
||||
return;
|
||||
}
|
||||
manager.grok_refresh_in_flight = true;
|
||||
ctx.spawn(
|
||||
async move { oauth::refresh_access_token(&refresh_token).await },
|
||||
|manager, result, ctx| {
|
||||
manager.grok_refresh_in_flight = false;
|
||||
match result {
|
||||
Ok(response) => {
|
||||
log::info!(
|
||||
"Refreshed Grok OAuth token (expires_in={:?}, has_refresh_token={})",
|
||||
response.expires_in,
|
||||
response.refresh_token.is_some(),
|
||||
);
|
||||
apply_grok_tokens(manager, response, ctx);
|
||||
}
|
||||
Err(err) => {
|
||||
// Leave the existing (possibly expired) token in place; the
|
||||
// server remains the authority and will reject it if it's
|
||||
// truly invalid. The request-time safety net
|
||||
// (`ApiKeyManager::refresh_grok_tokens_if_needed`) retries
|
||||
// on the next request.
|
||||
log::error!("Failed to refresh Grok OAuth token: {err:#}");
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
//! OAuth flow for connecting an xAI / Grok subscription (e.g. SuperGrok) to
|
||||
//! Warp, so users can "plug in" their subscription instead of pasting a
|
||||
//! pay-as-you-go API key.
|
||||
//!
|
||||
//! This mirrors the public Grok-CLI desktop OAuth flow: an OAuth 2.0
|
||||
//! Authorization Code grant with PKCE and a fixed loopback redirect URI. xAI's
|
||||
//! auth server only accepts the loopback redirect for an allowlisted
|
||||
//! `client_id` bound to a specific port, so we reuse the Grok-CLI client and
|
||||
//! bind the callback server to that exact port.
|
||||
//!
|
||||
//! Some browsers/networks can't reach the loopback callback (e.g. Private
|
||||
//! Network Access is blocked), in which case xAI's consent screen instead
|
||||
//! *displays* the authorization code for the user to paste back into the app.
|
||||
//! [`OauthAttempt::manual_code_exchange`] supports that fallback by capturing
|
||||
//! the attempt's PKCE verifier so a pasted code can be exchanged directly,
|
||||
//! without ever observing the loopback redirect.
|
||||
//!
|
||||
//! This module owns only the network/protocol side: building the authorize
|
||||
//! URL, running the loopback callback server, and exchanging/refreshing tokens
|
||||
//! at xAI's token endpoint. Persistence of the resulting tokens, proactive
|
||||
//! refresh scheduling, and injection into the request live in the parent
|
||||
//! [`crate::grok_subscription`] module (refresh orchestration) and
|
||||
//! [`crate::api_keys::ApiKeyManager`] (storage + request injection).
|
||||
|
||||
use std::io::{ErrorKind, Read, Write};
|
||||
use std::net::{Shutdown, TcpListener, TcpStream};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{bail, Context as _};
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use base64::Engine as _;
|
||||
// `std::time::Instant` is disallowed (no wasm support); `instant::Instant` is a
|
||||
// drop-in that re-exports the std type on native targets.
|
||||
use instant::Instant;
|
||||
use rand::RngCore as _;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const CLIENT_ID: &str = "b1a00492-073a-47ea-816f-4c329264a828";
|
||||
const AUTHORIZE_URL: &str = "https://auth.x.ai/oauth2/authorize";
|
||||
const TOKEN_URL: &str = "https://auth.x.ai/oauth2/token";
|
||||
const SCOPE: &str = "openid profile email offline_access grok-cli:access api:access";
|
||||
|
||||
const REDIRECT_HOST: &str = "127.0.0.1";
|
||||
const REDIRECT_PORT: u16 = 56121;
|
||||
|
||||
/// How long we keep the loopback server open waiting for the user to approve
|
||||
/// the consent screen in their browser.
|
||||
const CALLBACK_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
/// How long to nap between non-blocking `accept()` attempts.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// xAI's browser consent screen fetches the loopback callback from these
|
||||
/// origins. Since that request crosses origins (https://accounts.x.ai ->
|
||||
/// http://127.0.0.1), browsers require CORS and Private Network Access headers
|
||||
/// before the page can observe the callback response.
|
||||
const CORS_ALLOWED_ORIGINS: [&str; 2] = ["https://accounts.x.ai", "https://auth.x.ai"];
|
||||
|
||||
fn redirect_uri() -> String {
|
||||
format!("http://{REDIRECT_HOST}:{REDIRECT_PORT}/callback")
|
||||
}
|
||||
|
||||
/// One in-flight OAuth login attempt: the bound loopback callback listener
|
||||
/// plus the per-attempt PKCE/CSRF secrets, which never leave this module.
|
||||
///
|
||||
/// Construct with [`OauthAttempt::start`], open [`OauthAttempt::authorize_url`]
|
||||
/// in the browser, then await [`OauthAttempt::finish`] to obtain tokens. Tying
|
||||
/// the secrets to the attempt guarantees the same PKCE verifier and CSRF state
|
||||
/// are used for both the authorize URL and the code exchange.
|
||||
pub struct OauthAttempt {
|
||||
listener: TcpListener,
|
||||
pkce: PkceParams,
|
||||
}
|
||||
|
||||
impl OauthAttempt {
|
||||
/// Binds the loopback callback server and generates fresh per-attempt
|
||||
/// secrets. Call this before opening the browser so a bind failure (e.g.
|
||||
/// another login already in progress, or Grok-CLI holding the port)
|
||||
/// surfaces before a browser tab opens.
|
||||
pub fn start() -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
listener: bind_callback_listener()?,
|
||||
pkce: PkceParams::generate(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The authorization URL the user's browser should open to begin the flow.
|
||||
pub fn authorize_url(&self) -> String {
|
||||
authorize_url(&self.pkce)
|
||||
}
|
||||
|
||||
/// Runs the rest of the browser-based PKCE flow: waits for the loopback
|
||||
/// callback, validates the CSRF state, and exchanges the authorization
|
||||
/// code for tokens. Consumes the attempt so its secrets can't be reused.
|
||||
pub async fn finish(self) -> anyhow::Result<TokenResponse> {
|
||||
run_oauth_flow(self.listener, self.pkce).await
|
||||
}
|
||||
|
||||
/// Clones the PKCE verifier for the pasted-code fallback while the
|
||||
/// loopback flow continues racing in parallel.
|
||||
pub fn manual_code_exchange(&self) -> ManualCodeExchange {
|
||||
ManualCodeExchange {
|
||||
verifier: self.pkce.verifier.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Completes OAuth from a manually-pasted authorization code.
|
||||
///
|
||||
/// There is no redirect `state` to validate in this out-of-band path; PKCE
|
||||
/// protects the exchange.
|
||||
#[derive(Clone)]
|
||||
pub struct ManualCodeExchange {
|
||||
verifier: String,
|
||||
}
|
||||
|
||||
impl ManualCodeExchange {
|
||||
/// Exchanges a user-pasted authorization `code` with the attempt's PKCE verifier.
|
||||
pub async fn exchange(&self, code: &str) -> anyhow::Result<TokenResponse> {
|
||||
let code = code.trim();
|
||||
if code.is_empty() {
|
||||
bail!("enter the code shown in your browser to finish connecting");
|
||||
}
|
||||
exchange_code_for_tokens(code, &self.verifier).await
|
||||
}
|
||||
}
|
||||
|
||||
/// The per-attempt secrets for one authorization request: the PKCE
|
||||
/// verifier/challenge pair and the CSRF `state` value.
|
||||
struct PkceParams {
|
||||
verifier: String,
|
||||
challenge: String,
|
||||
/// CSRF token echoed back on the redirect and validated against the
|
||||
/// response before the code is exchanged.
|
||||
state: String,
|
||||
}
|
||||
|
||||
impl PkceParams {
|
||||
/// Generates a fresh PKCE verifier + S256 challenge and a random CSRF state.
|
||||
fn generate() -> Self {
|
||||
let verifier = random_url_safe_token();
|
||||
let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()));
|
||||
let state = random_url_safe_token();
|
||||
Self {
|
||||
verifier,
|
||||
challenge,
|
||||
state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a URL-safe, unpadded base64 string of 32 random bytes. This is used
|
||||
/// for both the PKCE code verifier (RFC 7636 allows 43-128 chars from the
|
||||
/// unreserved set) and the CSRF state.
|
||||
fn random_url_safe_token() -> String {
|
||||
let mut bytes = [0u8; 32];
|
||||
rand::thread_rng().fill_bytes(&mut bytes);
|
||||
URL_SAFE_NO_PAD.encode(bytes)
|
||||
}
|
||||
|
||||
/// Builds the authorization URL the user's browser should open to begin the
|
||||
/// flow.
|
||||
fn authorize_url(pkce: &PkceParams) -> String {
|
||||
let redirect = redirect_uri();
|
||||
// `plan=generic` opts the consent screen into xAI's generic OAuth plan tier
|
||||
// (required for loopback OAuth from non-allowlisted clients); `referrer`
|
||||
// is best-effort attribution in xAI's OAuth logs.
|
||||
let params: [(&str, &str); 9] = [
|
||||
("response_type", "code"),
|
||||
("client_id", CLIENT_ID),
|
||||
("redirect_uri", &redirect),
|
||||
("scope", SCOPE),
|
||||
("code_challenge", &pkce.challenge),
|
||||
("code_challenge_method", "S256"),
|
||||
("state", &pkce.state),
|
||||
("plan", "generic"),
|
||||
("referrer", "warp"),
|
||||
];
|
||||
let query =
|
||||
serde_urlencoded::to_string(params).expect("static OAuth params are always serializable");
|
||||
format!("{AUTHORIZE_URL}?{query}")
|
||||
}
|
||||
|
||||
/// The token endpoint's response. Fields beyond `access_token` are optional
|
||||
/// because xAI does not always return them. Other response fields (e.g.
|
||||
/// `token_type`, `scope`) are ignored since nothing consumes them.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TokenResponse {
|
||||
pub access_token: String,
|
||||
#[serde(default)]
|
||||
pub refresh_token: Option<String>,
|
||||
#[serde(default)]
|
||||
pub expires_in: Option<i64>,
|
||||
}
|
||||
|
||||
/// The authorization code and state captured from the loopback redirect.
|
||||
struct CallbackData {
|
||||
code: String,
|
||||
state: String,
|
||||
}
|
||||
|
||||
/// Binds the loopback callback server to the fixed redirect address.
|
||||
fn bind_callback_listener() -> anyhow::Result<TcpListener> {
|
||||
let listener = TcpListener::bind((REDIRECT_HOST, REDIRECT_PORT)).with_context(|| {
|
||||
format!(
|
||||
"couldn't bind the Grok OAuth callback server to {REDIRECT_HOST}:{REDIRECT_PORT}. \
|
||||
Another login may be in progress, or another app (e.g. Grok CLI) is using the port."
|
||||
)
|
||||
})?;
|
||||
listener
|
||||
.set_nonblocking(true)
|
||||
.context("failed to set the Grok OAuth callback listener to non-blocking mode")?;
|
||||
Ok(listener)
|
||||
}
|
||||
|
||||
/// Runs the full browser-based PKCE flow: waits for the loopback callback on a
|
||||
/// dedicated thread, validates the CSRF state, and exchanges the authorization
|
||||
/// code for tokens.
|
||||
async fn run_oauth_flow(listener: TcpListener, pkce: PkceParams) -> anyhow::Result<TokenResponse> {
|
||||
// The loopback accept loop is blocking, so run it on a dedicated OS thread
|
||||
// and bridge the result back through a runtime-agnostic async channel.
|
||||
let (tx, rx) = async_channel::bounded(1);
|
||||
std::thread::Builder::new()
|
||||
.name("grok-oauth-callback".to_owned())
|
||||
.spawn(move || {
|
||||
// `send_blocking` is disallowed (no wasm support); block this
|
||||
// dedicated thread on the async `send` instead.
|
||||
let _ = galaxyui_core::r#async::block_on(
|
||||
tx.send(wait_for_callback(&listener, CALLBACK_TIMEOUT)),
|
||||
);
|
||||
})
|
||||
.context("failed to spawn the Grok OAuth callback server thread")?;
|
||||
|
||||
let callback = rx
|
||||
.recv()
|
||||
.await
|
||||
.context("the Grok OAuth callback server stopped unexpectedly")??;
|
||||
|
||||
if callback.state != pkce.state {
|
||||
bail!("the authorization response state did not match — aborting to prevent CSRF");
|
||||
}
|
||||
|
||||
exchange_code_for_tokens(&callback.code, &pkce.verifier).await
|
||||
}
|
||||
|
||||
/// Blocks (on a non-blocking listener with polling) until the browser hits the
|
||||
/// redirect URI, returning the captured code and state, or an error on timeout.
|
||||
fn wait_for_callback(listener: &TcpListener, timeout: Duration) -> anyhow::Result<CallbackData> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
if Instant::now() >= deadline {
|
||||
bail!("timed out waiting for the Grok authorization callback");
|
||||
}
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => match handle_callback_connection(stream)? {
|
||||
Some(data) => return Ok(data),
|
||||
// Unrelated request (e.g. /favicon.ico); keep waiting.
|
||||
None => continue,
|
||||
},
|
||||
Err(ref e) if e.kind() == ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(anyhow::Error::new(e).context("Grok OAuth callback accept failed"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads a single HTTP request from the callback connection, writes back a
|
||||
/// minimal HTML response, and extracts the OAuth parameters.
|
||||
///
|
||||
/// Returns `Ok(None)` for requests that aren't the OAuth callback (so the
|
||||
/// caller keeps listening), `Ok(Some(..))` on a successful callback, and `Err`
|
||||
/// when the provider reported an error or the callback was malformed.
|
||||
fn handle_callback_connection(mut stream: TcpStream) -> anyhow::Result<Option<CallbackData>> {
|
||||
// The accepted stream may inherit the listener's non-blocking flag on some
|
||||
// platforms; force blocking reads with a timeout so we get the full request
|
||||
// line without spinning.
|
||||
stream.set_nonblocking(false).ok();
|
||||
stream.set_read_timeout(Some(Duration::from_secs(10))).ok();
|
||||
|
||||
let mut buf = [0u8; 8192];
|
||||
let n = stream
|
||||
.read(&mut buf)
|
||||
.context("failed to read the Grok OAuth callback request")?;
|
||||
let request = String::from_utf8_lossy(&buf[..n]);
|
||||
|
||||
let origin = request_header(&request, "Origin");
|
||||
|
||||
// The request line looks like: "GET /callback?code=...&state=... HTTP/1.1".
|
||||
let mut request_line_parts = request
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.split_whitespace();
|
||||
let method = request_line_parts.next().unwrap_or_default();
|
||||
let path = request_line_parts.next().unwrap_or_default();
|
||||
|
||||
if method == "OPTIONS" && path.starts_with("/callback") {
|
||||
write_response(&mut stream, "204 No Content", "", origin.as_deref());
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(query) = path
|
||||
.strip_prefix("/callback")
|
||||
.and_then(|rest| rest.strip_prefix('?'))
|
||||
else {
|
||||
write_response(
|
||||
&mut stream,
|
||||
"404 Not Found",
|
||||
"Not found.",
|
||||
origin.as_deref(),
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut code = None;
|
||||
let mut state = None;
|
||||
let mut error = None;
|
||||
let mut error_description = None;
|
||||
let pairs: Vec<(String, String)> = serde_urlencoded::from_str(query).unwrap_or_default();
|
||||
for (key, value) in pairs {
|
||||
match key.as_str() {
|
||||
"code" => code = Some(value),
|
||||
"state" => state = Some(value),
|
||||
"error" => error = Some(value),
|
||||
"error_description" => error_description = Some(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(error) = error {
|
||||
write_response(
|
||||
&mut stream,
|
||||
"400 Bad Request",
|
||||
FAILURE_HTML,
|
||||
origin.as_deref(),
|
||||
);
|
||||
let detail = error_description.unwrap_or(error);
|
||||
bail!("Grok authorization was denied or failed: {detail}");
|
||||
}
|
||||
|
||||
let (Some(code), Some(state)) = (code, state) else {
|
||||
write_response(
|
||||
&mut stream,
|
||||
"400 Bad Request",
|
||||
FAILURE_HTML,
|
||||
origin.as_deref(),
|
||||
);
|
||||
bail!("the Grok authorization callback was missing the code or state parameter");
|
||||
};
|
||||
write_response(&mut stream, "200 OK", SUCCESS_HTML, origin.as_deref());
|
||||
Ok(Some(CallbackData { code, state }))
|
||||
}
|
||||
fn request_header(request: &str, header_name: &str) -> Option<String> {
|
||||
request.lines().skip(1).find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case(header_name)
|
||||
.then(|| value.trim().to_owned())
|
||||
})
|
||||
}
|
||||
|
||||
/// Writes a minimal HTTP/1.1 response and closes the connection.
|
||||
fn write_response(stream: &mut TcpStream, status: &str, body: &str, origin: Option<&str>) {
|
||||
let cors_headers = cors_headers(origin);
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Type: text/html; charset=utf-8\r\n\
|
||||
{cors_headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = stream.shutdown(Shutdown::Both);
|
||||
}
|
||||
|
||||
fn cors_headers(origin: Option<&str>) -> String {
|
||||
origin
|
||||
.filter(|origin| CORS_ALLOWED_ORIGINS.contains(origin))
|
||||
.map(|origin| {
|
||||
format!(
|
||||
"Access-Control-Allow-Origin: {origin}\r\n\
|
||||
Access-Control-Allow-Methods: GET, OPTIONS\r\n\
|
||||
Access-Control-Allow-Headers: Content-Type\r\n\
|
||||
Access-Control-Allow-Private-Network: true\r\n\
|
||||
Vary: Origin\r\n"
|
||||
)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Exchanges the authorization code for OAuth tokens at xAI's token endpoint.
|
||||
async fn exchange_code_for_tokens(code: &str, verifier: &str) -> anyhow::Result<TokenResponse> {
|
||||
let redirect = redirect_uri();
|
||||
let form: [(&str, &str); 5] = [
|
||||
("grant_type", "authorization_code"),
|
||||
("code", code),
|
||||
("redirect_uri", &redirect),
|
||||
("client_id", CLIENT_ID),
|
||||
("code_verifier", verifier),
|
||||
];
|
||||
post_token_request(&form).await
|
||||
}
|
||||
|
||||
/// Exchanges a previously obtained refresh token for a fresh set of tokens via
|
||||
/// the OAuth 2.0 `refresh_token` grant. Used to keep the connected Grok
|
||||
/// subscription's access token valid without re-running the browser flow.
|
||||
///
|
||||
/// xAI may or may not return a new `refresh_token`; callers should fall back to
|
||||
/// the existing one when [`TokenResponse::refresh_token`] is `None` (rotation is
|
||||
/// optional in OAuth 2.0).
|
||||
pub async fn refresh_access_token(refresh_token: &str) -> anyhow::Result<TokenResponse> {
|
||||
let form: [(&str, &str); 3] = [
|
||||
("grant_type", "refresh_token"),
|
||||
("refresh_token", refresh_token),
|
||||
("client_id", CLIENT_ID),
|
||||
];
|
||||
post_token_request(&form).await
|
||||
}
|
||||
|
||||
/// POSTs a form-encoded body to xAI's token endpoint and parses the
|
||||
/// [`TokenResponse`]. Shared by the initial code exchange and refresh grants.
|
||||
async fn post_token_request<T: serde::Serialize + ?Sized>(
|
||||
form: &T,
|
||||
) -> anyhow::Result<TokenResponse> {
|
||||
let response = http_client::Client::new()
|
||||
.post(TOKEN_URL)
|
||||
.form(form)
|
||||
.send()
|
||||
.await
|
||||
.context("failed to send the Grok token request")?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
bail!("Grok token request failed ({status}): {body}");
|
||||
}
|
||||
|
||||
response
|
||||
.json::<TokenResponse>()
|
||||
.await
|
||||
.context("failed to parse the Grok token response")
|
||||
}
|
||||
|
||||
const SUCCESS_HTML: &str = "<!doctype html><html><head><meta charset=\"utf-8\">\
|
||||
<title>Warp — Grok connected</title></head>\
|
||||
<body style=\"font-family:system-ui,-apple-system,sans-serif;text-align:center;padding:3rem\">\
|
||||
<h1>Grok connected</h1><p>You can close this window and return to Warp.</p></body></html>";
|
||||
|
||||
const FAILURE_HTML: &str = "<!doctype html><html><head><meta charset=\"utf-8\">\
|
||||
<title>Warp — Grok authorization failed</title></head>\
|
||||
<body style=\"font-family:system-ui,-apple-system,sans-serif;text-align:center;padding:3rem\">\
|
||||
<h1>Authorization failed</h1><p>Something went wrong. Return to Warp and try again.</p></body></html>";
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "oauth_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,57 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn authorize_url_contains_required_params() {
|
||||
let pkce = PkceParams::generate();
|
||||
let url = authorize_url(&pkce);
|
||||
|
||||
assert!(url.starts_with("https://auth.x.ai/oauth2/authorize?"));
|
||||
assert!(url.contains("response_type=code"));
|
||||
assert!(url.contains(&format!("client_id={CLIENT_ID}")));
|
||||
assert!(url.contains("code_challenge_method=S256"));
|
||||
assert!(url.contains("scope=openid"));
|
||||
assert!(url.contains("plan=generic"));
|
||||
assert!(url.contains("referrer=warp"));
|
||||
// The redirect URI must be percent-encoded and match the registered value.
|
||||
assert!(url.contains("redirect_uri=http%3A%2F%2F127.0.0.1%3A56121%2Fcallback"));
|
||||
// The CSRF state and PKCE challenge are echoed into the URL verbatim
|
||||
// (both are URL-safe base64, so no percent-encoding is applied).
|
||||
assert!(url.contains(&format!("state={}", pkce.state)));
|
||||
assert!(url.contains(&format!("code_challenge={}", pkce.challenge)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_response_parses_minimal_and_full() {
|
||||
let minimal: TokenResponse =
|
||||
serde_json::from_str(r#"{"access_token":"abc"}"#).expect("minimal response should parse");
|
||||
assert_eq!(minimal.access_token, "abc");
|
||||
assert!(minimal.refresh_token.is_none());
|
||||
assert!(minimal.expires_in.is_none());
|
||||
|
||||
// Unconsumed response fields (token_type, scope) are ignored by serde.
|
||||
let full: TokenResponse = serde_json::from_str(
|
||||
r#"{"access_token":"a","refresh_token":"r","token_type":"Bearer","expires_in":3600,"scope":"api:access"}"#,
|
||||
)
|
||||
.expect("full response should parse");
|
||||
assert_eq!(full.access_token, "a");
|
||||
assert_eq!(full.refresh_token.as_deref(), Some("r"));
|
||||
assert_eq!(full.expires_in, Some(3600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_code_exchange_captures_attempt_verifier() {
|
||||
let pkce = PkceParams::generate();
|
||||
let exchange = ManualCodeExchange {
|
||||
verifier: pkce.verifier.clone(),
|
||||
};
|
||||
assert_eq!(exchange.verifier, pkce.verifier);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_code_exchange_rejects_blank_code() {
|
||||
let exchange = ManualCodeExchange {
|
||||
verifier: "verifier".to_string(),
|
||||
};
|
||||
let result = galaxyui_core::r#async::block_on(exchange.exchange(" "));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
@@ -7,14 +7,15 @@ cfg_if::cfg_if! {
|
||||
}
|
||||
}
|
||||
|
||||
use crate::index::{Entry, FileId};
|
||||
use ignore::gitignore::Gitignore;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use ignore::gitignore::Gitignore;
|
||||
use itertools::Itertools;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::index::{Entry, FileId};
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct FileSymbols {
|
||||
pub path: String,
|
||||
@@ -43,7 +44,7 @@ impl Outline {
|
||||
let mut queue = VecDeque::from([&self.root]);
|
||||
let mut repo_map = Vec::new();
|
||||
|
||||
// Iteratively print the files while perserving their traversal order.
|
||||
// Iteratively print the files while preserving their traversal order.
|
||||
while let Some(entry) = queue.pop_front() {
|
||||
match entry {
|
||||
Entry::Directory(directory) => {
|
||||
@@ -91,7 +92,7 @@ impl Outline {
|
||||
let mut queue = VecDeque::from([&self.root]);
|
||||
let mut file_to_symbols = HashMap::new();
|
||||
|
||||
// Iteratively print the files while perserving their traversal order.
|
||||
// Iteratively print the files while preserving their traversal order.
|
||||
while let Some(entry) = queue.pop_front() {
|
||||
match entry {
|
||||
Entry::Directory(directory) => {
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
use futures::channel::oneshot;
|
||||
use ignore::gitignore::Gitignore;
|
||||
use rayon::prelude::*;
|
||||
use repo_metadata::entry::is_file_parsable;
|
||||
use repo_metadata::RepositoryUpdate;
|
||||
use std::collections::HashMap;
|
||||
use std::{fs, path::Path};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use arborium::tree_sitter::{Parser, Query, QueryCursor, Tree};
|
||||
use futures::channel::oneshot;
|
||||
use ignore::gitignore::Gitignore;
|
||||
use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
use repo_metadata::entry::{is_file_parsable, BudgetExceededBehavior, IgnoredPathStrategy};
|
||||
use repo_metadata::RepositoryUpdate;
|
||||
use streaming_iterator::StreamingIterator;
|
||||
use syntax_tree::TextSlice;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
use crate::index::file_outline::{FileOutline, Outline, Symbol};
|
||||
use crate::index::THREADPOOL;
|
||||
use crate::index::{Entry, FileId, FileMetadata};
|
||||
use repo_metadata::entry::IgnoredPathStrategy;
|
||||
use crate::index::{Entry, FileId, FileMetadata, THREADPOOL};
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
@@ -55,6 +55,7 @@ pub async fn build_outline(
|
||||
MAX_DEPTH,
|
||||
0,
|
||||
&IgnoredPathStrategy::Exclude, // override_ignore_for_files
|
||||
BudgetExceededBehavior::StopAndLazyLoad,
|
||||
)?;
|
||||
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
@@ -231,7 +232,8 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
|
||||
if !is_file_parsable(path)? {
|
||||
return Err(anyhow!("File exceeds max file size limit for parsing"));
|
||||
}
|
||||
let Some(language) = languages::language_by_filename(path) else {
|
||||
let standardized_path = StandardizedPath::try_from_local(path)?;
|
||||
let Some(language) = languages::language_by_filename(&standardized_path) else {
|
||||
return Err(anyhow!("Language unsupported for file {:?}", path));
|
||||
};
|
||||
let content = fs::read_to_string(path)?;
|
||||
@@ -267,7 +269,11 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
|
||||
// of the allocator).
|
||||
//
|
||||
// See: https://github.com/tree-sitter/tree-sitter/issues/3129
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
|
||||
#[cfg(all(
|
||||
any(target_os = "linux", target_os = "freebsd"),
|
||||
target_env = "gnu",
|
||||
not(feature = "jemalloc")
|
||||
))]
|
||||
unsafe {
|
||||
nix::libc::malloc_trim(0);
|
||||
}
|
||||
@@ -333,199 +339,5 @@ fn get_symbols<'a>(
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn create_test_file(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
|
||||
let file_path = dir.path().join(filename);
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(content.as_bytes()).unwrap();
|
||||
file_path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_comments() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let content = r#"
|
||||
/// This is a struct for NewFunc
|
||||
struct NewFunc {
|
||||
a: str,
|
||||
}
|
||||
|
||||
// Hello
|
||||
// World
|
||||
fn first_function() {
|
||||
println!("First");
|
||||
}
|
||||
|
||||
impl NewFunc {
|
||||
fn second_function() {
|
||||
println!("Second");
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
|
||||
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols[0].name, "NewFunc");
|
||||
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
|
||||
assert_eq!(
|
||||
symbols[0].comment,
|
||||
Some(vec!["/// This is a struct for NewFunc".to_owned()])
|
||||
);
|
||||
assert_eq!(symbols[0].line_number, 3); // struct NewFunc is on line 3
|
||||
assert_eq!(symbols[1].name, "first_function");
|
||||
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[1].line_number, 9); // first_function is on line 9
|
||||
assert_eq!(symbols[2].name, "second_function");
|
||||
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[2].line_number, 14); // second_function is on line 14
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_multiple_languages() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let content = r#"
|
||||
struct NewFunc {
|
||||
a: str,
|
||||
}
|
||||
|
||||
fn first_function() {
|
||||
println!("First");
|
||||
}
|
||||
|
||||
impl NewFunc {
|
||||
fn second_function() {
|
||||
println!("Second");
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
|
||||
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 3);
|
||||
assert_eq!(symbols[0].name, "NewFunc");
|
||||
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
|
||||
assert_eq!(symbols[1].name, "first_function");
|
||||
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[2].name, "second_function");
|
||||
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
|
||||
|
||||
// Test parsing Python code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Regular function definitions (def keyword)
|
||||
// - Class definitions (class keyword)
|
||||
// - Method definitions within a class (def keyword)
|
||||
let python_content = r#"
|
||||
def first_function():
|
||||
print("First")
|
||||
|
||||
class TestClass:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def class_method(self):
|
||||
print("Method")
|
||||
|
||||
def second_function():
|
||||
print("Second")
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.py", python_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 5);
|
||||
assert_eq!(symbols[0].name, "first_function");
|
||||
assert_eq!(symbols[0].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestClass");
|
||||
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
|
||||
assert_eq!(symbols[2].name, "__init__");
|
||||
assert_eq!(symbols[2].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[3].name, "class_method");
|
||||
assert_eq!(symbols[3].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[4].name, "second_function");
|
||||
assert_eq!(symbols[4].type_prefix, Some("def".to_owned()));
|
||||
|
||||
// Test parsing JavaScript code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Function declarations
|
||||
// - Class declarations
|
||||
// - Method definitions
|
||||
// - Arrow functions assigned to variables
|
||||
let js_content = r#"
|
||||
function regularFunction() {
|
||||
console.log('Regular function');
|
||||
}
|
||||
|
||||
class TestClass {
|
||||
constructor() {
|
||||
this.value = 42;
|
||||
}
|
||||
|
||||
classMethod() {
|
||||
return this.value;
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.js", js_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 4);
|
||||
assert_eq!(symbols[0].name, "regularFunction");
|
||||
assert_eq!(symbols[0].type_prefix, Some("function".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestClass");
|
||||
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
|
||||
assert_eq!(symbols[2].name, "constructor");
|
||||
assert_eq!(symbols[2].type_prefix, None);
|
||||
assert_eq!(symbols[3].name, "classMethod");
|
||||
assert_eq!(symbols[3].type_prefix, None);
|
||||
|
||||
// Test parsing Go code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Function definitions (func keyword)
|
||||
// - Type definitions (struct, interface)
|
||||
// - Method definitions (func with receiver)
|
||||
let go_content = r#"
|
||||
package main
|
||||
|
||||
func mainFunction() {
|
||||
fmt.Println("Main function")
|
||||
}
|
||||
|
||||
type TestStruct struct {
|
||||
field string
|
||||
}
|
||||
|
||||
func (t *TestStruct) structMethod() string {
|
||||
return t.field
|
||||
}
|
||||
|
||||
type TestInterface interface {
|
||||
InterfaceMethod() string
|
||||
}
|
||||
|
||||
func helperFunction() {
|
||||
fmt.Println("Helper function")
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.go", go_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 5);
|
||||
assert_eq!(symbols[0].name, "mainFunction");
|
||||
assert_eq!(symbols[0].type_prefix, Some("func".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestStruct");
|
||||
assert_eq!(symbols[1].type_prefix, Some("type".to_owned()));
|
||||
assert_eq!(symbols[2].name, "structMethod");
|
||||
assert_eq!(symbols[2].type_prefix, Some("func".to_owned()));
|
||||
assert_eq!(symbols[3].name, "TestInterface");
|
||||
assert_eq!(symbols[3].type_prefix, Some("type".to_owned()));
|
||||
assert_eq!(symbols[4].name, "helperFunction");
|
||||
assert_eq!(symbols[4].type_prefix, Some("func".to_owned()));
|
||||
}
|
||||
}
|
||||
#[path = "native_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn create_test_file(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
|
||||
let file_path = dir.path().join(filename);
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(content.as_bytes()).unwrap();
|
||||
file_path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_comments() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let content = r#"
|
||||
/// This is a struct for NewFunc
|
||||
struct NewFunc {
|
||||
a: str,
|
||||
}
|
||||
|
||||
// Hello
|
||||
// World
|
||||
fn first_function() {
|
||||
println!("First");
|
||||
}
|
||||
|
||||
impl NewFunc {
|
||||
fn second_function() {
|
||||
println!("Second");
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
|
||||
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols[0].name, "NewFunc");
|
||||
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
|
||||
assert_eq!(
|
||||
symbols[0].comment,
|
||||
Some(vec!["/// This is a struct for NewFunc".to_owned()])
|
||||
);
|
||||
assert_eq!(symbols[0].line_number, 3); // struct NewFunc is on line 3
|
||||
assert_eq!(symbols[1].name, "first_function");
|
||||
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[1].line_number, 9); // first_function is on line 9
|
||||
assert_eq!(symbols[2].name, "second_function");
|
||||
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[2].line_number, 14); // second_function is on line 14
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_multiple_languages() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let content = r#"
|
||||
struct NewFunc {
|
||||
a: str,
|
||||
}
|
||||
|
||||
fn first_function() {
|
||||
println!("First");
|
||||
}
|
||||
|
||||
impl NewFunc {
|
||||
fn second_function() {
|
||||
println!("Second");
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.rs", content);
|
||||
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 3);
|
||||
assert_eq!(symbols[0].name, "NewFunc");
|
||||
assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
|
||||
assert_eq!(symbols[1].name, "first_function");
|
||||
assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
|
||||
assert_eq!(symbols[2].name, "second_function");
|
||||
assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
|
||||
|
||||
// Test parsing Python code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Regular function definitions (def keyword)
|
||||
// - Class definitions (class keyword)
|
||||
// - Method definitions within a class (def keyword)
|
||||
let python_content = r#"
|
||||
def first_function():
|
||||
print("First")
|
||||
|
||||
class TestClass:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def class_method(self):
|
||||
print("Method")
|
||||
|
||||
def second_function():
|
||||
print("Second")
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.py", python_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 5);
|
||||
assert_eq!(symbols[0].name, "first_function");
|
||||
assert_eq!(symbols[0].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestClass");
|
||||
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
|
||||
assert_eq!(symbols[2].name, "__init__");
|
||||
assert_eq!(symbols[2].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[3].name, "class_method");
|
||||
assert_eq!(symbols[3].type_prefix, Some("def".to_owned()));
|
||||
assert_eq!(symbols[4].name, "second_function");
|
||||
assert_eq!(symbols[4].type_prefix, Some("def".to_owned()));
|
||||
|
||||
// Test parsing JavaScript code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Function declarations
|
||||
// - Class declarations
|
||||
// - Method definitions
|
||||
// - Arrow functions assigned to variables
|
||||
let js_content = r#"
|
||||
function regularFunction() {
|
||||
console.log('Regular function');
|
||||
}
|
||||
|
||||
class TestClass {
|
||||
constructor() {
|
||||
this.value = 42;
|
||||
}
|
||||
|
||||
classMethod() {
|
||||
return this.value;
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.js", js_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 4);
|
||||
assert_eq!(symbols[0].name, "regularFunction");
|
||||
assert_eq!(symbols[0].type_prefix, Some("function".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestClass");
|
||||
assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
|
||||
assert_eq!(symbols[2].name, "constructor");
|
||||
assert_eq!(symbols[2].type_prefix, None);
|
||||
assert_eq!(symbols[3].name, "classMethod");
|
||||
assert_eq!(symbols[3].type_prefix, None);
|
||||
|
||||
// Test parsing Go code with multiple symbol definitions
|
||||
// This verifies parsing of:
|
||||
// - Function definitions (func keyword)
|
||||
// - Type definitions (struct, interface)
|
||||
// - Method definitions (func with receiver)
|
||||
let go_content = r#"
|
||||
package main
|
||||
|
||||
func mainFunction() {
|
||||
fmt.Println("Main function")
|
||||
}
|
||||
|
||||
type TestStruct struct {
|
||||
field string
|
||||
}
|
||||
|
||||
func (t *TestStruct) structMethod() string {
|
||||
return t.field
|
||||
}
|
||||
|
||||
type TestInterface interface {
|
||||
InterfaceMethod() string
|
||||
}
|
||||
|
||||
func helperFunction() {
|
||||
fmt.Println("Helper function")
|
||||
}
|
||||
"#;
|
||||
let file_path = create_test_file(&temp_dir, "multiple.go", go_content);
|
||||
let outline = parse_file_outline(&file_path).unwrap();
|
||||
let symbols = outline.symbols.unwrap();
|
||||
assert_eq!(symbols.len(), 5);
|
||||
assert_eq!(symbols[0].name, "mainFunction");
|
||||
assert_eq!(symbols[0].type_prefix, Some("func".to_owned()));
|
||||
assert_eq!(symbols[1].name, "TestStruct");
|
||||
assert_eq!(symbols[1].type_prefix, Some("type".to_owned()));
|
||||
assert_eq!(symbols[2].name, "structMethod");
|
||||
assert_eq!(symbols[2].type_prefix, Some("func".to_owned()));
|
||||
assert_eq!(symbols[3].name, "TestInterface");
|
||||
assert_eq!(symbols[3].type_prefix, Some("type".to_owned()));
|
||||
assert_eq!(symbols[4].name, "helperFunction");
|
||||
assert_eq!(symbols[4].type_prefix, Some("func".to_owned()));
|
||||
}
|
||||
@@ -46,5 +46,5 @@ impl ChangedFiles {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "changed_files_test.rs"]
|
||||
#[path = "changed_files_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::*;
|
||||
|
||||
// Helper function to create a PathBuf from a string
|
||||
fn pb(path: &str) -> PathBuf {
|
||||
PathBuf::from(path)
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::path::Path;
|
||||
|
||||
use string_offset::ByteOffset;
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
mod naive;
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
@@ -99,7 +101,8 @@ pub fn chunk_code<'a>(code: &'a str, path: &'a Path) -> Vec<Fragment<'a>> {
|
||||
/// could not be chunked for any reason.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
fn try_chunk_code_semantically<'a>(code: &'a str, path: &'a Path) -> Option<Vec<Fragment<'a>>> {
|
||||
let language = languages::language_by_filename(path)?;
|
||||
let standardized_path = StandardizedPath::try_from_local(path).ok()?;
|
||||
let language = languages::language_by_filename(&standardized_path)?;
|
||||
semantic::chunk_code(code, path, MAX_BYTES_PER_CHUNK, &language.grammar).ok()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use crate::index::full_source_code_embedding::chunker::{coalesce_fragments, Fragment};
|
||||
use std::path::Path;
|
||||
|
||||
use itertools::Itertools;
|
||||
use line_span::{LineSpan, LineSpans};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::index::full_source_code_embedding::chunker::{coalesce_fragments, Fragment};
|
||||
|
||||
/// Chunks the given file into [`Fragment`]s. Each chunk is at most `num_lines_per_chunk` lines long, and contains at most `max_bytes_per_chunk` bytes.
|
||||
pub(super) fn chunk_code<'a>(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_chunker() {
|
||||
let code = "This is some text content\nthat should be chunked\nusing the naive chunker\nbecause the language isn't recognized.";
|
||||
|
||||
@@ -48,7 +48,11 @@ pub(super) fn chunk_code<'a>(
|
||||
// of the allocator).
|
||||
//
|
||||
// See: https://github.com/tree-sitter/tree-sitter/issues/3129
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
|
||||
#[cfg(all(
|
||||
any(target_os = "linux", target_os = "freebsd"),
|
||||
target_env = "gnu",
|
||||
not(feature = "jemalloc")
|
||||
))]
|
||||
unsafe {
|
||||
nix::libc::malloc_trim(0);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::path::Path;
|
||||
|
||||
use languages::language_by_filename;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -33,12 +34,14 @@ fn main() {
|
||||
"#;
|
||||
|
||||
let max_chunk_size = 128;
|
||||
let language_path =
|
||||
StandardizedPath::try_new("/test.rs").expect("test path should be absolute");
|
||||
|
||||
let chunks = chunk_code(
|
||||
source_code,
|
||||
Path::new("test.rs"),
|
||||
max_chunk_size,
|
||||
&language_by_filename(Path::new("test.rs"))
|
||||
&language_by_filename(&language_path)
|
||||
.expect("Rust language must exist")
|
||||
.grammar,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use async_channel;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -5,32 +11,31 @@ use futures::stream::AbortHandle;
|
||||
use galaxy_core::safe_error;
|
||||
use galaxyui::{Entity, ModelContext, ModelHandle};
|
||||
use ignore::gitignore::Gitignore;
|
||||
use instant::Instant;
|
||||
#[cfg(feature = "local_fs")]
|
||||
use repo_metadata::entry::IgnoredPathStrategy;
|
||||
use repo_metadata::entry::{BudgetExceededBehavior, IgnoredPathStrategy};
|
||||
use repo_metadata::Repository;
|
||||
use std::{path::Path, sync::Arc};
|
||||
use galaxyui_core::{Entity, ModelContext, ModelHandle};
|
||||
|
||||
use super::fragment_metadata::{
|
||||
FragmentMetadata, LeafToFragmentMetadata, LeafToFragmentMetadataUpdates,
|
||||
};
|
||||
use super::manager::{
|
||||
CodebaseIndexFinishedStatus, CodebaseIndexStatus, FragmentMetadataLookupError,
|
||||
RetrieveFileError,
|
||||
};
|
||||
use super::merkle_tree::{MerkleTree, SerializedCodebaseIndex};
|
||||
#[cfg(feature = "local_fs")]
|
||||
use super::search_shaping::build_fragments_from_file_contents;
|
||||
use super::search_shaping::{fragments_to_context_locations, ReadFragmentResult};
|
||||
use super::store_client::StoreClient;
|
||||
use super::sync_client::{FlushFragmentResult, SyncOperationError};
|
||||
use super::{
|
||||
fragment_metadata::{FragmentMetadata, LeafToFragmentMetadata, LeafToFragmentMetadataUpdates},
|
||||
manager::{CodebaseIndexFinishedStatus, CodebaseIndexStatus, RetrieveFileError},
|
||||
merkle_tree::{MerkleTree, SerializedCodebaseIndex},
|
||||
store_client::StoreClient,
|
||||
sync_client::{FlushFragmentResult, SyncOperationError},
|
||||
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
|
||||
};
|
||||
use crate::{
|
||||
index::locations::{CodeContextLocation, FileFragmentLocation},
|
||||
telemetry::{AITelemetryEvent, CodebaseContextSyncType},
|
||||
workspace::{WorkspaceMetadata, WorkspaceMetadataEvent},
|
||||
};
|
||||
use instant::Instant;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
ops::Range,
|
||||
path::PathBuf,
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
time::Duration,
|
||||
};
|
||||
use crate::index::locations::CodeContextLocation;
|
||||
use crate::telemetry::{AITelemetryEvent, CodebaseContextSyncType};
|
||||
use crate::workspace::{WorkspaceMetadata, WorkspaceMetadataEvent};
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
@@ -44,12 +49,11 @@ cfg_if::cfg_if! {
|
||||
Entry,
|
||||
matches_gitignores,
|
||||
full_source_code_embedding::sync_client::CodebaseIndexSyncOperation,
|
||||
full_source_code_embedding::FragmentLocation
|
||||
};
|
||||
use galaxy_core::send_telemetry_from_ctx;
|
||||
use galaxy_core::interval_timer::IntervalTimer;
|
||||
use galaxyui::r#async::Timer;
|
||||
use galaxyui::SingletonEntity;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
use galaxyui_core::SingletonEntity;
|
||||
use galaxy_core::sync_queue::SyncQueue;
|
||||
use sha2::Digest;
|
||||
}
|
||||
@@ -316,7 +320,9 @@ pub enum CodebaseIndexEvent {
|
||||
retrieval_id: RetrievalID,
|
||||
error: Error,
|
||||
},
|
||||
SyncStateUpdated,
|
||||
SyncStateUpdated {
|
||||
root_path: PathBuf,
|
||||
},
|
||||
IndexMetadataUpdated {
|
||||
root_path: PathBuf,
|
||||
event: WorkspaceMetadataEvent,
|
||||
@@ -498,6 +504,13 @@ impl CodebaseIndex {
|
||||
store_client: Arc<dyn StoreClient>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
if self
|
||||
.pending_file_changes
|
||||
.as_ref()
|
||||
.is_none_or(|changed_files| changed_files.is_empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let last_server_synced_root_node = self.last_server_synced_root_node();
|
||||
let old_state = self.update_tree_sync_state(
|
||||
TreeSourceSyncState::Syncing {
|
||||
@@ -876,7 +889,9 @@ impl CodebaseIndex {
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> TreeSourceSyncState {
|
||||
let old_state = std::mem::replace(&mut self.tree_sync_state, new_state);
|
||||
ctx.emit(CodebaseIndexEvent::SyncStateUpdated);
|
||||
ctx.emit(CodebaseIndexEvent::SyncStateUpdated {
|
||||
root_path: self.repo_path.clone(),
|
||||
});
|
||||
old_state
|
||||
}
|
||||
|
||||
@@ -885,7 +900,9 @@ impl CodebaseIndex {
|
||||
if let TreeSourceSyncState::Syncing { sync_progress, .. } = &mut self.tree_sync_state {
|
||||
*sync_progress = Some(progress);
|
||||
|
||||
ctx.emit(CodebaseIndexEvent::SyncStateUpdated);
|
||||
ctx.emit(CodebaseIndexEvent::SyncStateUpdated {
|
||||
root_path: self.repo_path.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -920,6 +937,9 @@ impl CodebaseIndex {
|
||||
// First traverse the repo path to retrieve all files we want to parse.
|
||||
let mut files = Vec::new();
|
||||
let mut remaining_file_quotas = max_num_files_limit;
|
||||
// Codebase embedding must not operate on a partial tree: the file limit
|
||||
// is an intentional cost cap, so exceeding it fails the build rather
|
||||
// than silently indexing a breadth-first subset of the repository.
|
||||
let entry = Entry::build_tree(
|
||||
&repo_path,
|
||||
&mut files,
|
||||
@@ -928,6 +948,7 @@ impl CodebaseIndex {
|
||||
MAX_DEPTH,
|
||||
0,
|
||||
&IgnoredPathStrategy::Exclude, // override_ignore_for_files
|
||||
BudgetExceededBehavior::FailFast,
|
||||
)?;
|
||||
|
||||
Ok(BuildFileTreeResult {
|
||||
@@ -1318,6 +1339,30 @@ impl CodebaseIndex {
|
||||
self.leaf_node_to_fragment_metadatas.get(leaf_hash.as_ref())
|
||||
}
|
||||
|
||||
pub(super) fn fragment_metadatas_from_hashes(
|
||||
&self,
|
||||
root_hash: &NodeHash,
|
||||
content_hashes: &[ContentHash],
|
||||
) -> Result<HashMap<ContentHash, Vec<FragmentMetadata>>, FragmentMetadataLookupError> {
|
||||
let current_root_hash = self
|
||||
.last_server_synced_root_node()
|
||||
.ok_or(FragmentMetadataLookupError::IndexNotSynced)?;
|
||||
if ¤t_root_hash != root_hash {
|
||||
return Err(FragmentMetadataLookupError::RootHashMismatch {
|
||||
requested: root_hash.clone(),
|
||||
current: current_root_hash,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(content_hashes
|
||||
.iter()
|
||||
.filter_map(|hash| {
|
||||
self.fragment_metadatas_from_hash(hash)
|
||||
.map(|metadata| (hash.clone(), metadata.clone()))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn repo_metadata(&self) -> RepoMetadata {
|
||||
RepoMetadata {
|
||||
path: Some(self.repo_path.to_string_lossy().to_string()),
|
||||
@@ -1342,12 +1387,20 @@ impl CodebaseIndex {
|
||||
}
|
||||
|
||||
pub(super) fn codebase_index_status(&self) -> CodebaseIndexStatus {
|
||||
let has_synced_version = self.last_server_synced_root_node().is_some();
|
||||
let root_hash = self.last_server_synced_root_node();
|
||||
let has_synced_version = root_hash.is_some();
|
||||
#[cfg(feature = "local_fs")]
|
||||
let has_pending_file_changes = self
|
||||
.pending_file_changes
|
||||
.as_ref()
|
||||
.is_some_and(|changes| !changes.is_empty());
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
let has_pending_file_changes = false;
|
||||
match &self.tree_sync_state {
|
||||
TreeSourceSyncState::Synced {
|
||||
server_sync_result, ..
|
||||
} => CodebaseIndexStatus {
|
||||
has_pending: false,
|
||||
has_pending: has_pending_file_changes,
|
||||
has_synced_version,
|
||||
last_sync_successful: Some(match server_sync_result {
|
||||
ServerSyncResult::Success => CodebaseIndexFinishedStatus::Completed,
|
||||
@@ -1356,18 +1409,21 @@ impl CodebaseIndex {
|
||||
}
|
||||
}),
|
||||
sync_progress: None,
|
||||
root_hash: root_hash.clone(),
|
||||
},
|
||||
TreeSourceSyncState::InitializeTreeFailure(e) => CodebaseIndexStatus {
|
||||
has_pending: false,
|
||||
has_synced_version,
|
||||
last_sync_successful: Some(CodebaseIndexFinishedStatus::Failed(e.into())),
|
||||
sync_progress: None,
|
||||
root_hash: root_hash.clone(),
|
||||
},
|
||||
TreeSourceSyncState::Syncing { sync_progress, .. } => CodebaseIndexStatus {
|
||||
has_pending: true,
|
||||
has_synced_version,
|
||||
last_sync_successful: None,
|
||||
sync_progress: *sync_progress,
|
||||
root_hash: root_hash.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1622,82 +1678,21 @@ impl CodebaseIndex {
|
||||
}
|
||||
}
|
||||
|
||||
// Convert fragments into CodeContextLocations. This function groups and dedupes fragments in the same file.
|
||||
// It also allows the caller to define a context line number surrounding the relevant fragment.
|
||||
fn process_fragments(
|
||||
&self,
|
||||
fragments: Vec<Fragment>,
|
||||
context_lines: usize,
|
||||
) -> HashSet<CodeContextLocation> {
|
||||
// Map to collect fragments by file path
|
||||
let mut fragments_by_path: HashMap<&PathBuf, Vec<Range<usize>>> = HashMap::new();
|
||||
let mut whole_files = HashSet::new();
|
||||
|
||||
// First pass - collect all fragments and their line ranges by file path
|
||||
for fragment in &fragments {
|
||||
if let Some(metadata) = self
|
||||
.fragment_metadatas_from_hash(&fragment.content_hash)
|
||||
.and_then(|metadatas| {
|
||||
metadatas.iter().find(|m| {
|
||||
m.absolute_path == fragment.location.absolute_path
|
||||
&& m.location.byte_range == fragment.location.byte_range
|
||||
})
|
||||
})
|
||||
{
|
||||
// Add line range with context to the appropriate file's collection
|
||||
let path = &fragment.location.absolute_path;
|
||||
let start = metadata.location.start_line.saturating_sub(context_lines);
|
||||
let end = metadata.location.end_line + 1 + context_lines; // Make the range inclusive on both ends
|
||||
|
||||
fragments_by_path.entry(path).or_default().push(start..end);
|
||||
} else {
|
||||
// Fallback to whole file if metadata not found
|
||||
whole_files.insert(fragment.location.absolute_path.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass - process each file's fragments
|
||||
let mut result = HashSet::new();
|
||||
|
||||
// Process each file's fragments
|
||||
for (path, mut line_ranges) in fragments_by_path {
|
||||
if line_ranges.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// We can skip the fragments if the entire file is already included in the context.
|
||||
if whole_files.contains(path) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sort ranges by start position
|
||||
line_ranges.sort_by_key(|range| range.start);
|
||||
|
||||
// Merge overlapping or adjacent ranges
|
||||
let mut merged_ranges: Vec<Range<usize>> = Vec::new();
|
||||
for range in line_ranges {
|
||||
if let Some(last) = merged_ranges.last_mut() {
|
||||
// If current range overlaps or is adjacent to the last one, merge them
|
||||
if range.start <= last.end {
|
||||
last.end = last.end.max(range.end);
|
||||
} else {
|
||||
merged_ranges.push(range);
|
||||
}
|
||||
} else {
|
||||
merged_ranges.push(range);
|
||||
}
|
||||
}
|
||||
|
||||
// Add file fragment location with all merged ranges
|
||||
result.insert(CodeContextLocation::Fragment(FileFragmentLocation {
|
||||
path: path.clone(),
|
||||
line_ranges: merged_ranges,
|
||||
}));
|
||||
}
|
||||
|
||||
// Add whole files to the result set
|
||||
result.extend(whole_files.into_iter().map(CodeContextLocation::WholeFile));
|
||||
result
|
||||
// Keep local and remote search aligned by using the same fragment-to-context expansion
|
||||
// helper for range merging, deduping, and context-line handling.
|
||||
fragments_to_context_locations(
|
||||
fragments,
|
||||
|content_hash| {
|
||||
self.fragment_metadatas_from_hash(content_hash)
|
||||
.map(Vec::as_slice)
|
||||
},
|
||||
context_lines,
|
||||
)
|
||||
}
|
||||
|
||||
/// A new index built from a snapshot. This constructor builds the index and starts
|
||||
@@ -2100,13 +2095,14 @@ impl CodebaseIndex {
|
||||
match entry.and_then(|entry| dunce::canonicalize(entry.path())) {
|
||||
Ok(child_path) => {
|
||||
// Ignore paths that are excluded by .gitignore, end with .git, or are symlinks.
|
||||
if matches_gitignores(
|
||||
&child_path,
|
||||
is_dir,
|
||||
&*gitignores,
|
||||
false, /* check_ancestors */
|
||||
) || child_path.ends_with(".git")
|
||||
if child_path.ends_with(".git")
|
||||
|| child_path.is_symlink()
|
||||
|| matches_gitignores(
|
||||
&child_path,
|
||||
child_path.is_dir(),
|
||||
&*gitignores,
|
||||
false, /* check_ancestors */
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -2244,13 +2240,14 @@ impl CodebaseIndex {
|
||||
match entry.and_then(|entry| dunce::canonicalize(entry.path())) {
|
||||
Ok(child_path) => {
|
||||
// Ignore paths that are excluded by .gitignore, end with .git, or are symlinks.
|
||||
if matches_gitignores(
|
||||
&child_path,
|
||||
is_dir,
|
||||
&*gitignores,
|
||||
false, /* check_ancestors */
|
||||
) || child_path.ends_with(".git")
|
||||
if child_path.ends_with(".git")
|
||||
|| child_path.is_symlink()
|
||||
|| matches_gitignores(
|
||||
&child_path,
|
||||
child_path.is_dir(),
|
||||
&*gitignores,
|
||||
false, /* check_ancestors */
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -2320,98 +2317,22 @@ impl CodebaseIndex {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ReadFragmentResult {
|
||||
pub successfully_read: Vec<Fragment>,
|
||||
pub fail_to_read: Vec<ContentHash>,
|
||||
pub fail_to_read_path: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
pub(super) async fn build_fragments_from_metadata(
|
||||
metadatas: impl IntoIterator<Item = (ContentHash, FragmentMetadata)>,
|
||||
) -> ReadFragmentResult {
|
||||
let mut fragments = Vec::new();
|
||||
let mut fail_to_read = Vec::new();
|
||||
let mut fail_to_read_path = Vec::new();
|
||||
|
||||
// Group fragments by file path
|
||||
let mut fragments_by_path: HashMap<_, Vec<_>> = HashMap::new();
|
||||
for (content_hash, metadata) in metadatas {
|
||||
fragments_by_path
|
||||
.entry(metadata.absolute_path)
|
||||
.or_default()
|
||||
.push((content_hash, metadata.location.byte_range));
|
||||
}
|
||||
|
||||
// Process each file and its fragments
|
||||
for (file_path, file_fragments) in fragments_by_path {
|
||||
let mut has_failed_to_read_fragments = false;
|
||||
// Read the file content once
|
||||
if let Ok(file_content) = async_fs::read_to_string(&file_path).await {
|
||||
// Process all fragments for this file
|
||||
for (content_hash, fragment_ranges) in file_fragments {
|
||||
let start_idx = fragment_ranges.start.as_usize();
|
||||
let end_idx = fragment_ranges.end.as_usize();
|
||||
|
||||
if start_idx <= end_idx
|
||||
&& end_idx <= file_content.len()
|
||||
&& file_content.is_char_boundary(start_idx)
|
||||
&& file_content.is_char_boundary(end_idx)
|
||||
{
|
||||
let content = file_content[start_idx..end_idx].to_string();
|
||||
if content.is_empty() {
|
||||
log::trace!(
|
||||
"Fragment for {:?} with range {:?} is empty",
|
||||
file_path.display(),
|
||||
fragment_ranges
|
||||
);
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
} else if ContentHash::from_content(&content) != content_hash {
|
||||
log::trace!(
|
||||
"Fragment for {:?} with range {:?} does not match its content hash",
|
||||
file_path.display(),
|
||||
fragment_ranges
|
||||
);
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
} else {
|
||||
fragments.push(Fragment {
|
||||
content,
|
||||
content_hash,
|
||||
location: FragmentLocation {
|
||||
absolute_path: file_path.clone(),
|
||||
byte_range: fragment_ranges,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else {
|
||||
log::trace!("Invalid byte range {fragment_ranges:?} for file: {file_path:?}");
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log::trace!("Failed to read file: {file_path:?}");
|
||||
fail_to_read.extend(
|
||||
file_fragments
|
||||
.into_iter()
|
||||
.map(|(content_hash, _)| content_hash),
|
||||
);
|
||||
has_failed_to_read_fragments = true;
|
||||
}
|
||||
|
||||
if has_failed_to_read_fragments {
|
||||
fail_to_read_path.push(file_path);
|
||||
let metadatas = metadatas.into_iter().collect::<Vec<_>>();
|
||||
let mut file_contents = HashMap::new();
|
||||
for path in metadatas
|
||||
.iter()
|
||||
.map(|(_, metadata)| metadata.absolute_path.clone())
|
||||
.collect::<HashSet<_>>()
|
||||
{
|
||||
if let Ok(file_content) = async_fs::read_to_string(&path).await {
|
||||
file_contents.insert(path, file_content);
|
||||
}
|
||||
}
|
||||
|
||||
ReadFragmentResult {
|
||||
successfully_read: fragments,
|
||||
fail_to_read,
|
||||
fail_to_read_path,
|
||||
}
|
||||
build_fragments_from_file_contents(metadatas, &file_contents)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
|
||||
@@ -1,34 +1,32 @@
|
||||
#![allow(clippy::single_range_in_vec_init)]
|
||||
use chrono::Utc;
|
||||
use string_offset::ByteOffset;
|
||||
use virtual_fs::{Stub, VirtualFS};
|
||||
|
||||
use crate::index::full_source_code_embedding::changed_files::ChangedFiles;
|
||||
use crate::index::full_source_code_embedding::codebase_index::MAX_DEPTH;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
FragmentLocation, LeafToFragmentMetadata,
|
||||
};
|
||||
|
||||
use crate::index::full_source_code_embedding::merkle_tree::MerkleHash;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::MerkleTree;
|
||||
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
|
||||
use crate::index::full_source_code_embedding::{
|
||||
ContentHash, EmbeddingConfig, Fragment, FragmentMetadata,
|
||||
};
|
||||
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
|
||||
use futures::executor::block_on;
|
||||
use galaxy_util::standardized_path::StandardizedPath;
|
||||
use galaxyui::{App, SingletonEntity};
|
||||
use repo_metadata::DirectoryWatcher;
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::executor::block_on;
|
||||
use repo_metadata::DirectoryWatcher;
|
||||
use string_offset::ByteOffset;
|
||||
use virtual_fs::{Stub, VirtualFS};
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
use galaxyui_core::{App, SingletonEntity};
|
||||
|
||||
use super::{
|
||||
CodebaseIndex, CodebaseIndexTimeStampMetadata, TreeSourceSyncState,
|
||||
CodebaseIndex, CodebaseIndexTimeStampMetadata, ServerSyncResult, TreeSourceSyncState,
|
||||
DEFAULT_INCREMENAL_SYNC_FLUSH_INTERVAL,
|
||||
};
|
||||
use crate::index::full_source_code_embedding::changed_files::ChangedFiles;
|
||||
use crate::index::full_source_code_embedding::codebase_index::MAX_DEPTH;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
FragmentLocation, LeafToFragmentMetadata,
|
||||
};
|
||||
use crate::index::full_source_code_embedding::merkle_tree::{MerkleHash, MerkleTree};
|
||||
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
|
||||
use crate::index::full_source_code_embedding::{
|
||||
ContentHash, EmbeddingConfig, Fragment, FragmentMetadata,
|
||||
};
|
||||
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
|
||||
|
||||
impl CodebaseIndex {
|
||||
fn new_for_test(
|
||||
@@ -104,6 +102,86 @@ fn create_test_metadata(
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synced_index_with_queued_file_changes_reports_pending_status() {
|
||||
VirtualFS::test(
|
||||
"synced_index_with_queued_file_changes_reports_pending_status",
|
||||
|dirs, mut sandbox| {
|
||||
App::test((), |mut app| async move {
|
||||
app.add_singleton_model(DirectoryWatcher::new);
|
||||
|
||||
let repo_name = "warp-virtual";
|
||||
sandbox.mkdir(repo_name);
|
||||
sandbox.with_files(vec![Stub::FileWithContent(
|
||||
format!("{repo_name}/existing_file").as_str(),
|
||||
"existing content",
|
||||
)]);
|
||||
|
||||
let repo_path = dunce::canonicalize(dirs.tests().join(repo_name)).unwrap();
|
||||
let build_file_tree_result =
|
||||
block_on(CodebaseIndex::build_file_tree(repo_path.clone(), None)).unwrap();
|
||||
let (tree, _) =
|
||||
block_on(MerkleTree::try_new(build_file_tree_result.file_tree)).unwrap();
|
||||
|
||||
let mut index = CodebaseIndex::new_for_test(Default::default(), &mut app);
|
||||
index.tree_sync_state = TreeSourceSyncState::Synced {
|
||||
tree,
|
||||
server_sync_result: ServerSyncResult::Success,
|
||||
};
|
||||
|
||||
let mut changed_files = ChangedFiles::default();
|
||||
changed_files.upsertions.insert(repo_path.join("new_file"));
|
||||
index.pending_file_changes = Some(changed_files);
|
||||
|
||||
let status = index.codebase_index_status();
|
||||
assert!(status.has_pending());
|
||||
assert!(status.has_synced_version());
|
||||
assert_eq!(status.last_sync_successful(), Some(true));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synced_index_without_pending_file_changes_stays_ready_after_flush() {
|
||||
VirtualFS::test(
|
||||
"synced_index_without_pending_file_changes_stays_ready_after_flush",
|
||||
|dirs, mut sandbox| {
|
||||
App::test((), |mut app| async move {
|
||||
app.add_singleton_model(DirectoryWatcher::new);
|
||||
|
||||
let repo_name = "warp-virtual";
|
||||
sandbox.mkdir(repo_name);
|
||||
sandbox.with_files(vec![Stub::FileWithContent(
|
||||
format!("{repo_name}/existing_file").as_str(),
|
||||
"existing content",
|
||||
)]);
|
||||
|
||||
let repo_path = dunce::canonicalize(dirs.tests().join(repo_name)).unwrap();
|
||||
let build_file_tree_result =
|
||||
block_on(CodebaseIndex::build_file_tree(repo_path, None)).unwrap();
|
||||
let (tree, _) =
|
||||
block_on(MerkleTree::try_new(build_file_tree_result.file_tree)).unwrap();
|
||||
|
||||
let mut test_index = CodebaseIndex::new_for_test(Default::default(), &mut app);
|
||||
test_index.tree_sync_state = TreeSourceSyncState::Synced {
|
||||
tree,
|
||||
server_sync_result: ServerSyncResult::Success,
|
||||
};
|
||||
let index = app.add_model(|_| test_index);
|
||||
|
||||
index.update(&mut app, |index, ctx| {
|
||||
index.flush_pending_file_changes(ctx);
|
||||
|
||||
let status = index.codebase_index_status();
|
||||
assert!(!status.has_pending());
|
||||
assert!(status.has_synced_version());
|
||||
assert_eq!(status.last_sync_successful(), Some(true));
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn test_empty_fragments() {
|
||||
App::test((), |mut app| async move {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use std::{collections::HashMap, ops::Range, path::PathBuf};
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use itertools::Itertools;
|
||||
use repo_metadata::{BuildTreeError, DirectoryWatcher, Repository};
|
||||
@@ -13,12 +11,12 @@ cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
use chrono::Utc;
|
||||
use super::changed_files::ChangedFiles;
|
||||
use crate::index::path_passes_filters;
|
||||
use crate::index::{is_git_internal_path, matches_gitignores};
|
||||
use ignore::gitignore::Gitignore;
|
||||
use notify_debouncer_full::notify::{RecursiveMode, WatchFilter};
|
||||
use galaxy_core::features::FeatureFlag;
|
||||
use watcher::{BulkFilesystemWatcher, BulkFilesystemWatcherEvent};
|
||||
use galaxyui::r#async::Timer;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
use galaxy_core::{send_telemetry_from_ctx, report_if_error};
|
||||
use crate::telemetry::AITelemetryEvent;
|
||||
use instant::Instant;
|
||||
@@ -27,21 +25,16 @@ cfg_if::cfg_if! {
|
||||
}
|
||||
}
|
||||
use galaxy_core::safe_anyhow;
|
||||
use galaxyui::{AppContext, Entity, ModelContext, ModelHandle, SingletonEntity};
|
||||
use galaxyui_core::{AppContext, Entity, ModelContext, ModelHandle, SingletonEntity};
|
||||
|
||||
use super::{
|
||||
codebase_index::{CodebaseIndexEvent, RetrievalID, SyncProgress},
|
||||
fragment_metadata::FragmentMetadata,
|
||||
priority_queue::{BuildQueue, Priority},
|
||||
snapshot::*,
|
||||
store_client::StoreClient,
|
||||
CodebaseIndex, EmbeddingConfig, Error as CodebaseIndexError, NodeHash,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
index::locations::CodeContextLocation,
|
||||
workspace::{WorkspaceMetadata, WorkspaceMetadataEvent},
|
||||
};
|
||||
use super::codebase_index::{CodebaseIndexEvent, RetrievalID, SyncProgress};
|
||||
use super::fragment_metadata::FragmentMetadata;
|
||||
use super::priority_queue::{BuildQueue, Priority};
|
||||
use super::snapshot::*;
|
||||
use super::store_client::StoreClient;
|
||||
use super::{CodebaseIndex, ContentHash, EmbeddingConfig, Error as CodebaseIndexError, NodeHash};
|
||||
use crate::index::locations::CodeContextLocation;
|
||||
use crate::workspace::{WorkspaceMetadata, WorkspaceMetadataEvent};
|
||||
|
||||
/// The interval for debouncing filesystem events.
|
||||
const REPO_WATCHER_DEBOUNCE_DURATION: Duration = Duration::from_secs(10);
|
||||
@@ -69,6 +62,19 @@ pub enum RetrieveFileError {
|
||||
IndexNotFound,
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum FragmentMetadataLookupError {
|
||||
#[error("Codebase index not found")]
|
||||
IndexNotFound,
|
||||
#[error("Codebase index has no synced root hash")]
|
||||
IndexNotSynced,
|
||||
#[error("Codebase index root hash mismatch: requested {requested}, current {current}")]
|
||||
RootHashMismatch {
|
||||
requested: NodeHash,
|
||||
current: NodeHash,
|
||||
},
|
||||
}
|
||||
|
||||
pub enum CodebaseIndexManagerEvent {
|
||||
RetrievalRequestCompleted {
|
||||
retrieval_id: RetrievalID,
|
||||
@@ -79,7 +85,9 @@ pub enum CodebaseIndexManagerEvent {
|
||||
retrieval_id: RetrievalID,
|
||||
error_message: String,
|
||||
},
|
||||
SyncStateUpdated,
|
||||
SyncStateUpdated {
|
||||
root_path: PathBuf,
|
||||
},
|
||||
IndexMetadataUpdated {
|
||||
root_path: PathBuf,
|
||||
event: WorkspaceMetadataEvent,
|
||||
@@ -87,7 +95,9 @@ pub enum CodebaseIndexManagerEvent {
|
||||
RemoveExpiredIndexMetadata {
|
||||
expired_metadata: Arc<Vec<PathBuf>>,
|
||||
},
|
||||
NewIndexCreated,
|
||||
NewIndexCreated {
|
||||
root_path: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
/// User-facing indexing errors.
|
||||
@@ -132,6 +142,7 @@ pub struct CodebaseIndexStatus {
|
||||
pub(super) has_synced_version: bool,
|
||||
pub(super) last_sync_successful: Option<CodebaseIndexFinishedStatus>,
|
||||
pub(super) sync_progress: Option<SyncProgress>,
|
||||
pub(super) root_hash: Option<NodeHash>,
|
||||
}
|
||||
|
||||
impl CodebaseIndexStatus {
|
||||
@@ -156,17 +167,127 @@ impl CodebaseIndexStatus {
|
||||
pub fn sync_progress(&self) -> Option<&SyncProgress> {
|
||||
self.sync_progress.as_ref()
|
||||
}
|
||||
|
||||
pub fn root_hash(&self) -> Option<&NodeHash> {
|
||||
self.root_hash.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
struct CodebaseIndexStatusEventKey {
|
||||
has_pending: bool,
|
||||
has_synced_version: bool,
|
||||
last_sync_status: Option<CodebaseIndexFinishedStatusEventKey>,
|
||||
sync_progress: Option<SyncProgressEventKey>,
|
||||
root_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl From<&CodebaseIndexStatus> for CodebaseIndexStatusEventKey {
|
||||
fn from(status: &CodebaseIndexStatus) -> Self {
|
||||
Self {
|
||||
has_pending: status.has_pending,
|
||||
has_synced_version: status.has_synced_version,
|
||||
last_sync_status: status
|
||||
.last_sync_successful
|
||||
.as_ref()
|
||||
.map(CodebaseIndexFinishedStatusEventKey::from),
|
||||
sync_progress: status
|
||||
.sync_progress
|
||||
.as_ref()
|
||||
.map(SyncProgressEventKey::from),
|
||||
root_hash: status.root_hash.as_ref().map(ToString::to_string),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
enum CodebaseIndexFinishedStatusEventKey {
|
||||
Completed,
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
impl From<&CodebaseIndexFinishedStatus> for CodebaseIndexFinishedStatusEventKey {
|
||||
fn from(status: &CodebaseIndexFinishedStatus) -> Self {
|
||||
match status {
|
||||
CodebaseIndexFinishedStatus::Completed => Self::Completed,
|
||||
CodebaseIndexFinishedStatus::Failed(error) => Self::Failed(error.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
enum SyncProgressEventKey {
|
||||
Discovering {
|
||||
total_nodes: usize,
|
||||
},
|
||||
Syncing {
|
||||
completed_nodes: usize,
|
||||
total_nodes: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<&SyncProgress> for SyncProgressEventKey {
|
||||
fn from(progress: &SyncProgress) -> Self {
|
||||
match progress {
|
||||
SyncProgress::Discovering { total_nodes } => Self::Discovering {
|
||||
total_nodes: *total_nodes,
|
||||
},
|
||||
SyncProgress::Syncing {
|
||||
completed_nodes,
|
||||
total_nodes,
|
||||
} => Self::Syncing {
|
||||
completed_nodes: *completed_nodes,
|
||||
total_nodes: *total_nodes,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
pub enum BuildSource<'a> {
|
||||
FromPath(&'a Path),
|
||||
FromPersistedMetadata(WorkspaceMetadata),
|
||||
}
|
||||
pub struct CodebaseIndexManagerConfig {
|
||||
persisted_index_metadata: Vec<WorkspaceMetadata>,
|
||||
max_index_count: Option<usize>,
|
||||
max_files_repo_limit: usize,
|
||||
embedding_generation_batch_size: usize,
|
||||
store_client: Arc<dyn StoreClient>,
|
||||
indexing_enabled: bool,
|
||||
restore_persisted_indices_on_startup: bool,
|
||||
}
|
||||
|
||||
impl CodebaseIndexManagerConfig {
|
||||
pub fn new(
|
||||
persisted_index_metadata: Vec<WorkspaceMetadata>,
|
||||
max_index_count: Option<usize>,
|
||||
max_files_repo_limit: usize,
|
||||
embedding_generation_batch_size: usize,
|
||||
store_client: Arc<dyn StoreClient>,
|
||||
indexing_enabled: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
persisted_index_metadata,
|
||||
max_index_count,
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
store_client,
|
||||
indexing_enabled,
|
||||
restore_persisted_indices_on_startup: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn defer_persisted_index_restore(mut self) -> Self {
|
||||
self.restore_persisted_indices_on_startup = false;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Manager for the codebase index states across the app.
|
||||
pub struct CodebaseIndexManager {
|
||||
codebase_indices: HashMap<PathBuf, ModelHandle<CodebaseIndex>>,
|
||||
|
||||
last_emitted_codebase_index_statuses: HashMap<PathBuf, CodebaseIndexStatusEventKey>,
|
||||
|
||||
store_client: Arc<dyn StoreClient>,
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
@@ -179,6 +300,11 @@ pub struct CodebaseIndexManager {
|
||||
max_files_repo_limit: usize,
|
||||
|
||||
embedding_generation_batch_size: usize,
|
||||
|
||||
indexing_enabled: bool,
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
snapshot_storage: Option<SnapshotStorage>,
|
||||
}
|
||||
|
||||
impl CodebaseIndexManager {
|
||||
@@ -189,25 +315,100 @@ impl CodebaseIndexManager {
|
||||
max_files_repo_limit: usize,
|
||||
embedding_generation_batch_size: usize,
|
||||
store_client: Arc<dyn StoreClient>,
|
||||
indexing_enabled: bool,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> Self {
|
||||
let config = CodebaseIndexManagerConfig::new(
|
||||
persisted_index_metadata,
|
||||
max_index_count,
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
store_client,
|
||||
indexing_enabled,
|
||||
);
|
||||
Self::new_with_config(config, ctx)
|
||||
}
|
||||
|
||||
#[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
|
||||
pub fn new_with_config(
|
||||
config: CodebaseIndexManagerConfig,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> Self {
|
||||
#[cfg(feature = "local_fs")]
|
||||
{
|
||||
report_if_error!(migrate_snapshots_to_secure_dir_if_needed());
|
||||
Self::new_with_snapshot_storage(config, SnapshotStorage::app_default(), ctx)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
{
|
||||
Self::new_internal(config, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
pub fn new_with_snapshot_storage(
|
||||
config: CodebaseIndexManagerConfig,
|
||||
snapshot_storage: Option<SnapshotStorage>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> Self {
|
||||
Self::new_internal(config, snapshot_storage, ctx)
|
||||
}
|
||||
|
||||
#[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
|
||||
fn new_internal(
|
||||
config: CodebaseIndexManagerConfig,
|
||||
#[cfg(feature = "local_fs")] snapshot_storage: Option<SnapshotStorage>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> Self {
|
||||
let CodebaseIndexManagerConfig {
|
||||
persisted_index_metadata,
|
||||
max_index_count,
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
store_client,
|
||||
indexing_enabled,
|
||||
restore_persisted_indices_on_startup,
|
||||
} = config;
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
let file_watcher = ctx.add_model(|ctx| BulkFilesystemWatcher::new(REPO_WATCHER_DEBOUNCE_DURATION, ctx));
|
||||
ctx.subscribe_to_model(&file_watcher, Self::handle_watcher_event);
|
||||
}
|
||||
}
|
||||
if !indexing_enabled {
|
||||
log::debug!(
|
||||
"Codebase indexing disabled for this launch mode; skipping restore of {:?} persisted codebase indices",
|
||||
persisted_index_metadata.len()
|
||||
);
|
||||
|
||||
return Self {
|
||||
codebase_indices: HashMap::new(),
|
||||
last_emitted_codebase_index_statuses: HashMap::new(),
|
||||
store_client,
|
||||
#[cfg(feature = "local_fs")]
|
||||
watcher: file_watcher,
|
||||
build_queue: BuildQueue::empty(),
|
||||
max_indices: max_index_count,
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
indexing_enabled,
|
||||
#[cfg(feature = "local_fs")]
|
||||
snapshot_storage,
|
||||
};
|
||||
}
|
||||
|
||||
log::debug!(
|
||||
"Received {:?} persisted codebase indices",
|
||||
persisted_index_metadata.len()
|
||||
);
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
report_if_error!(migrate_snapshots_to_secure_dir_if_needed());
|
||||
|
||||
let (invalid_metadata, valid_metadata) =
|
||||
split_snapshot_metadata_by_validity(persisted_index_metadata);
|
||||
let (invalid_metadata, valid_metadata) = split_snapshot_metadata_by_validity(
|
||||
persisted_index_metadata,
|
||||
snapshot_storage.as_ref(),
|
||||
);
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
let (invalid_metadata, valid_metadata) = (persisted_index_metadata, Vec::new());
|
||||
|
||||
ctx.emit(CodebaseIndexManagerEvent::RemoveExpiredIndexMetadata {
|
||||
expired_metadata: Arc::new(
|
||||
@@ -217,16 +418,18 @@ impl CodebaseIndexManager {
|
||||
.collect(),
|
||||
),
|
||||
});
|
||||
|
||||
if let Some(snapshot_file_dir) = snapshot_dir() {
|
||||
clean_up_snapshot_files(&snapshot_file_dir, &valid_metadata);
|
||||
#[cfg(feature = "local_fs")]
|
||||
if let Some(snapshot_storage) = snapshot_storage.as_ref() {
|
||||
clean_up_snapshot_files(snapshot_storage.path(), &valid_metadata);
|
||||
}
|
||||
|
||||
// For the moment, we've decided to load all snapshots regardless of the index count.
|
||||
let build_queue = BuildQueue::new_with_persisted(valid_metadata);
|
||||
let build_queue =
|
||||
BuildQueue::new_with_persisted(valid_metadata, restore_persisted_indices_on_startup);
|
||||
|
||||
let mut me = Self {
|
||||
codebase_indices: HashMap::new(),
|
||||
last_emitted_codebase_index_statuses: HashMap::new(),
|
||||
store_client,
|
||||
#[cfg(feature = "local_fs")]
|
||||
watcher: file_watcher,
|
||||
@@ -234,12 +437,12 @@ impl CodebaseIndexManager {
|
||||
max_indices: max_index_count,
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
indexing_enabled,
|
||||
#[cfg(feature = "local_fs")]
|
||||
snapshot_storage,
|
||||
};
|
||||
|
||||
// Start building the first index in the queue.
|
||||
if let Some(next_repo) = me.build_queue.pick_next_sync() {
|
||||
me.build_and_sync_codebase_index(BuildSource::FromPersistedMetadata(next_repo), ctx);
|
||||
}
|
||||
me.start_next_queued_index(ctx);
|
||||
|
||||
me
|
||||
}
|
||||
@@ -250,6 +453,7 @@ impl CodebaseIndexManager {
|
||||
let file_watcher = ctx.add_model(|_| BulkFilesystemWatcher::new_for_test());
|
||||
Self {
|
||||
codebase_indices: HashMap::new(),
|
||||
last_emitted_codebase_index_statuses: HashMap::new(),
|
||||
store_client,
|
||||
#[cfg(feature = "local_fs")]
|
||||
watcher: file_watcher,
|
||||
@@ -257,6 +461,9 @@ impl CodebaseIndexManager {
|
||||
max_indices: None,
|
||||
max_files_repo_limit: 0,
|
||||
embedding_generation_batch_size: 100,
|
||||
indexing_enabled: true,
|
||||
#[cfg(feature = "local_fs")]
|
||||
snapshot_storage: SnapshotStorage::app_default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,8 +513,15 @@ impl CodebaseIndexManager {
|
||||
|
||||
// Remove snapshots from disk.
|
||||
let to_drop_clone = to_drop.clone();
|
||||
#[cfg(feature = "local_fs")]
|
||||
let snapshot_storage = self.snapshot_storage.clone();
|
||||
ctx.spawn(
|
||||
async move { Self::drop_index_snapshots(to_drop_clone).await },
|
||||
async move {
|
||||
#[cfg(feature = "local_fs")]
|
||||
Self::drop_index_snapshots(snapshot_storage, to_drop_clone).await;
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
let _ = to_drop_clone;
|
||||
},
|
||||
|_, _, _| {},
|
||||
);
|
||||
|
||||
@@ -317,11 +531,15 @@ impl CodebaseIndexManager {
|
||||
});
|
||||
}
|
||||
|
||||
/// Remove the gien index snapshots from disk.
|
||||
async fn drop_index_snapshots(to_drop: Vec<PathBuf>) {
|
||||
if let Some(snapshot_dir) = snapshot_dir() {
|
||||
/// Remove the given index snapshots from disk.
|
||||
#[cfg(feature = "local_fs")]
|
||||
async fn drop_index_snapshots(
|
||||
snapshot_storage: Option<SnapshotStorage>,
|
||||
to_drop: Vec<PathBuf>,
|
||||
) {
|
||||
if let Some(snapshot_storage) = snapshot_storage {
|
||||
for codebase_root in &to_drop {
|
||||
Self::drop_index_snapshot(&snapshot_dir, codebase_root).await;
|
||||
Self::drop_index_snapshot(snapshot_storage.path(), codebase_root).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -345,6 +563,7 @@ impl CodebaseIndexManager {
|
||||
|
||||
// Drop the in-memory index.
|
||||
self.codebase_indices.remove(root_path);
|
||||
self.last_emitted_codebase_index_statuses.remove(root_path);
|
||||
|
||||
// Stop the filewatcher from receiving events for this codebase.
|
||||
#[cfg(feature = "local_fs")]
|
||||
@@ -359,11 +578,16 @@ impl CodebaseIndexManager {
|
||||
|
||||
// Remove snapshot from disk.
|
||||
let root_path_clone = root_path.clone();
|
||||
#[cfg(feature = "local_fs")]
|
||||
let snapshot_storage = self.snapshot_storage.clone();
|
||||
ctx.spawn(
|
||||
async move {
|
||||
if let Some(snapshot_dir) = snapshot_dir() {
|
||||
Self::drop_index_snapshot(&snapshot_dir, &root_path_clone).await;
|
||||
#[cfg(feature = "local_fs")]
|
||||
if let Some(snapshot_storage) = snapshot_storage {
|
||||
Self::drop_index_snapshot(snapshot_storage.path(), &root_path_clone).await;
|
||||
}
|
||||
#[cfg(not(feature = "local_fs"))]
|
||||
let _ = root_path_clone;
|
||||
},
|
||||
|_, _, _| {},
|
||||
);
|
||||
@@ -451,6 +675,7 @@ impl CodebaseIndexManager {
|
||||
#[cfg(feature = "local_fs")]
|
||||
fn handle_watcher_event(
|
||||
&mut self,
|
||||
_: ModelHandle<BulkFilesystemWatcher>,
|
||||
event: &BulkFilesystemWatcherEvent,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
@@ -462,6 +687,9 @@ impl CodebaseIndexManager {
|
||||
}
|
||||
|
||||
pub fn handle_active_session_changed(&mut self, active_directory: &Path) {
|
||||
if !self.is_indexing_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(root_path) = self.root_path_for_codebase(active_directory) else {
|
||||
return;
|
||||
};
|
||||
@@ -517,11 +745,17 @@ impl CodebaseIndexManager {
|
||||
|
||||
/// Ensures the current number of indices is below the maximum.
|
||||
pub fn can_create_new_indices(&self) -> bool {
|
||||
if !self.is_indexing_enabled() {
|
||||
return false;
|
||||
}
|
||||
self.max_indices
|
||||
.is_none_or(|max_indices| self.codebase_indices.len() < max_indices)
|
||||
}
|
||||
|
||||
pub fn handle_session_bootstrapped(&mut self, working_directory: &Path) {
|
||||
if !self.is_indexing_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(root_path) = self.root_path_for_codebase(working_directory) else {
|
||||
return;
|
||||
};
|
||||
@@ -553,6 +787,22 @@ impl CodebaseIndexManager {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn fragment_metadatas_from_hashes(
|
||||
&self,
|
||||
repo_path: &Path,
|
||||
root_hash: &NodeHash,
|
||||
content_hashes: &[ContentHash],
|
||||
app: &AppContext,
|
||||
) -> Result<HashMap<ContentHash, Vec<FragmentMetadata>>, FragmentMetadataLookupError> {
|
||||
let (codebase_index, _) = self
|
||||
.get_codebase_index_internal(repo_path)
|
||||
.map_err(|_| FragmentMetadataLookupError::IndexNotFound)?;
|
||||
|
||||
codebase_index
|
||||
.as_ref(app)
|
||||
.fragment_metadatas_from_hashes(root_hash, content_hashes)
|
||||
}
|
||||
|
||||
pub fn get_codebase_paths(&self) -> impl Iterator<Item = &PathBuf> {
|
||||
self.codebase_indices.keys()
|
||||
}
|
||||
@@ -561,13 +811,37 @@ impl CodebaseIndexManager {
|
||||
self.codebase_indices.len()
|
||||
}
|
||||
|
||||
pub fn index_directory(&mut self, directory: PathBuf, ctx: &mut ModelContext<Self>) {
|
||||
let directory = dunce::canonicalize(&directory).unwrap_or(directory);
|
||||
if !self.codebase_indices.contains_key(&directory) {
|
||||
self.build_and_sync_codebase_index(BuildSource::FromPath(&directory), ctx);
|
||||
// Starting a new codebase index should be considered into sync state updates.
|
||||
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated);
|
||||
pub fn is_indexing_enabled(&self) -> bool {
|
||||
self.indexing_enabled
|
||||
}
|
||||
|
||||
pub fn start_persisted_index_restore(&mut self, ctx: &mut ModelContext<Self>) {
|
||||
if !self.is_indexing_enabled() {
|
||||
return;
|
||||
}
|
||||
if self.build_queue.start() {
|
||||
self.start_next_queued_index(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_directory(&mut self, directory: PathBuf, ctx: &mut ModelContext<Self>) -> bool {
|
||||
if !self.is_indexing_enabled() {
|
||||
return false;
|
||||
}
|
||||
if self.root_path_for_codebase(&directory).is_none() {
|
||||
if !self.build_and_sync_codebase_index(BuildSource::FromPath(&directory), ctx) {
|
||||
return false;
|
||||
}
|
||||
let indexed_directory = self
|
||||
.root_path_for_codebase(&directory)
|
||||
.unwrap_or_else(|| directory.clone());
|
||||
self.record_codebase_index_status(&indexed_directory, ctx);
|
||||
// Starting a new codebase index should be considered into sync state updates.
|
||||
ctx.emit(CodebaseIndexManagerEvent::NewIndexCreated {
|
||||
root_path: indexed_directory,
|
||||
});
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
@@ -577,9 +851,21 @@ impl CodebaseIndexManager {
|
||||
gitignores: Arc<Vec<Gitignore>>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
let watch_filter = WatchFilter::with_filter(Arc::new(move |path| {
|
||||
path_passes_filters(path, gitignores.as_slice())
|
||||
}));
|
||||
// The codebase indexer only cares about source files:
|
||||
// skip anything inside `.git/` and anything matched by gitignore
|
||||
// (including descendants of an ignored ancestor directory).
|
||||
// The same predicate gates both directory descent and event emission.
|
||||
let filter = Arc::new(move |path: &Path| {
|
||||
!is_git_internal_path(path)
|
||||
&& !matches_gitignores(
|
||||
path,
|
||||
path.is_dir(),
|
||||
gitignores.as_slice(),
|
||||
true, /* check_ancestors */
|
||||
)
|
||||
});
|
||||
|
||||
let watch_filter = WatchFilter::with_filter(filter.clone(), filter);
|
||||
self.watcher.update(ctx, |watcher, _ctx| {
|
||||
std::mem::drop(watcher.register_path(
|
||||
root_path,
|
||||
@@ -607,9 +893,12 @@ impl CodebaseIndexManager {
|
||||
&mut self,
|
||||
build_source: BuildSource,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
) -> bool {
|
||||
if !self.is_indexing_enabled() {
|
||||
return false;
|
||||
}
|
||||
if !self.can_create_new_indices() {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
let repo_path = match build_source {
|
||||
@@ -624,7 +913,7 @@ impl CodebaseIndexManager {
|
||||
Ok(path) => path,
|
||||
Err(e) => {
|
||||
log::error!("Failed to canonicalize repository path: {e:?}");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -635,7 +924,7 @@ impl CodebaseIndexManager {
|
||||
Ok(handle) => handle,
|
||||
Err(e) => {
|
||||
log::error!("Failed to start tracking repository: {e:?}");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -646,11 +935,15 @@ impl CodebaseIndexManager {
|
||||
.codebase_indices
|
||||
.entry(canonical_key)
|
||||
.or_insert_with(|| {
|
||||
#[cfg(feature = "local_fs")]
|
||||
let snapshot_storage = self.snapshot_storage.clone();
|
||||
let index = Self::build_and_sync_codebase_index_internal(
|
||||
self.store_client.clone(),
|
||||
handle,
|
||||
self.max_files_repo_limit,
|
||||
self.embedding_generation_batch_size,
|
||||
#[cfg(feature = "local_fs")]
|
||||
snapshot_storage,
|
||||
ctx,
|
||||
);
|
||||
|
||||
@@ -666,6 +959,7 @@ impl CodebaseIndexManager {
|
||||
index.update_timestamps_from_metadata(metadata);
|
||||
});
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Checks whether a snapshot exists for the index and attempts to load it;
|
||||
@@ -675,6 +969,7 @@ impl CodebaseIndexManager {
|
||||
repository: ModelHandle<Repository>,
|
||||
max_files_repo_limit: usize,
|
||||
embedding_generation_batch_size: usize,
|
||||
#[cfg(feature = "local_fs")] snapshot_storage: Option<SnapshotStorage>,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> ModelHandle<CodebaseIndex> {
|
||||
let codebase_index = ctx.add_model(|ctx| {
|
||||
@@ -684,13 +979,17 @@ impl CodebaseIndexManager {
|
||||
.as_ref(ctx)
|
||||
.root_dir()
|
||||
.to_local_path()
|
||||
.is_some_and(|p| has_snapshot(&p))
|
||||
.is_some_and(|p| {
|
||||
snapshot_storage
|
||||
.as_ref()
|
||||
.is_some_and(|storage| storage.has_snapshot(&p))
|
||||
})
|
||||
{
|
||||
if let Some(snapshot_dir) = snapshot_dir() {
|
||||
if let Some(snapshot_storage) = snapshot_storage.as_ref() {
|
||||
let read_snapshot_start_time = Instant::now();
|
||||
match read_snapshot(
|
||||
store_client.clone(),
|
||||
snapshot_dir.as_path(),
|
||||
snapshot_storage.path(),
|
||||
repository.clone(),
|
||||
max_files_repo_limit,
|
||||
embedding_generation_batch_size,
|
||||
@@ -733,6 +1032,7 @@ impl CodebaseIndexManager {
|
||||
|
||||
fn handle_codebase_index_event(
|
||||
&mut self,
|
||||
_: ModelHandle<CodebaseIndex>,
|
||||
event: &CodebaseIndexEvent,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) {
|
||||
@@ -753,8 +1053,8 @@ impl CodebaseIndexManager {
|
||||
fragments: fragments.clone(),
|
||||
out_of_sync_delay: *out_of_sync_delay,
|
||||
}),
|
||||
CodebaseIndexEvent::SyncStateUpdated => {
|
||||
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated)
|
||||
CodebaseIndexEvent::SyncStateUpdated { root_path } => {
|
||||
self.maybe_emit_sync_state_updated(root_path, ctx);
|
||||
}
|
||||
CodebaseIndexEvent::IndexMetadataUpdated { root_path, event } => {
|
||||
ctx.emit(CodebaseIndexManagerEvent::IndexMetadataUpdated {
|
||||
@@ -785,11 +1085,42 @@ impl CodebaseIndexManager {
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_emit_sync_state_updated(&mut self, root_path: &Path, ctx: &mut ModelContext<Self>) {
|
||||
if self.record_codebase_index_status(root_path, ctx) {
|
||||
ctx.emit(CodebaseIndexManagerEvent::SyncStateUpdated {
|
||||
root_path: root_path.to_path_buf(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn record_codebase_index_status(
|
||||
&mut self,
|
||||
root_path: &Path,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> bool {
|
||||
let root_path = dunce::canonicalize(root_path).unwrap_or_else(|_| root_path.to_path_buf());
|
||||
let Some(status) = self.get_codebase_index_status_for_path(root_path.as_path(), ctx) else {
|
||||
return false;
|
||||
};
|
||||
let key = CodebaseIndexStatusEventKey::from(&status);
|
||||
match self.last_emitted_codebase_index_statuses.get(&root_path) {
|
||||
Some(previous_key) if previous_key == &key => false,
|
||||
Some(_) | None => {
|
||||
self.last_emitted_codebase_index_statuses
|
||||
.insert(root_path, key);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn on_index_build_finished(&mut self, finished_repo: &Path, ctx: &mut ModelContext<Self>) {
|
||||
let Ok(_) = self.get_codebase_index_internal(finished_repo) else {
|
||||
return;
|
||||
};
|
||||
self.start_next_queued_index(ctx);
|
||||
}
|
||||
|
||||
fn start_next_queued_index(&mut self, ctx: &mut ModelContext<Self>) {
|
||||
if let Some(next_repo) = self.build_queue.pick_next_sync() {
|
||||
self.build_and_sync_codebase_index(BuildSource::FromPersistedMetadata(next_repo), ctx);
|
||||
}
|
||||
@@ -810,6 +1141,20 @@ impl CodebaseIndexManager {
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub fn with_indexed_codebase<T>(
|
||||
&mut self,
|
||||
path: &Path,
|
||||
on_found: impl FnOnce(&mut Self, &Path, &mut ModelContext<Self>) -> T,
|
||||
on_missing: impl FnOnce(&mut Self, &Path, &mut ModelContext<Self>) -> T,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> T {
|
||||
let Some(indexed_repo_path) = self.root_path_for_codebase(path) else {
|
||||
return on_missing(self, path, ctx);
|
||||
};
|
||||
|
||||
on_found(self, indexed_repo_path.as_path(), ctx)
|
||||
}
|
||||
|
||||
fn get_codebase_index_internal(
|
||||
&self,
|
||||
path: &Path,
|
||||
@@ -909,15 +1254,15 @@ impl CodebaseIndexManager {
|
||||
}
|
||||
};
|
||||
|
||||
let snapshot_dir = match snapshot_dir() {
|
||||
Some(dir) => dir,
|
||||
let snapshot_storage = match self.snapshot_storage.as_ref() {
|
||||
Some(storage) => storage,
|
||||
None => {
|
||||
log::warn!("No snapshot directory to write to");
|
||||
Self::schedule_next_snapshot_write(repo_path, ctx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let snapshot_path = snapshot_path(&snapshot_dir, repo_path.as_path());
|
||||
let snapshot_path = snapshot_storage.snapshot_path(repo_path.as_path());
|
||||
|
||||
// Update timestamp eagerly so concurrent calls to has_unsnapshotted_changes()
|
||||
// won't trigger a duplicate snapshot while the background write is in progress.
|
||||
@@ -974,6 +1319,9 @@ impl CodebaseIndexManager {
|
||||
directory_path: &Path,
|
||||
ctx: &mut ModelContext<Self>,
|
||||
) -> anyhow::Result<()> {
|
||||
if !self.is_indexing_enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
// Find the root path for this directory's codebase
|
||||
let Some(repo_path) = self.root_path_for_codebase(directory_path) else {
|
||||
return Err(anyhow::anyhow!("Failed to find root path for directory"));
|
||||
@@ -987,7 +1335,7 @@ impl CodebaseIndexManager {
|
||||
codebase_index.update(ctx, |index, _ctx| {
|
||||
// Check if the index is in a state where it can perform incremental updates
|
||||
let status = index.codebase_index_status();
|
||||
if status.has_pending {
|
||||
if status.last_sync_successful() != Some(true) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1013,3 +1361,7 @@ impl Entity for CodebaseIndexManager {
|
||||
}
|
||||
|
||||
impl SingletonEntity for CodebaseIndexManager {}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "manager_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
use chrono::Utc;
|
||||
#[cfg(feature = "local_fs")]
|
||||
use repo_metadata::DirectoryWatcher;
|
||||
use galaxyui_core::App;
|
||||
|
||||
use super::{
|
||||
BuildSource, CodebaseIndexFinishedStatus, CodebaseIndexManager, CodebaseIndexManagerConfig,
|
||||
CodebaseIndexStatus, CodebaseIndexStatusEventKey, CodebaseIndexingError, SyncProgress,
|
||||
};
|
||||
use crate::index::full_source_code_embedding::store_client::MockStoreClient;
|
||||
#[cfg(feature = "local_fs")]
|
||||
use crate::index::full_source_code_embedding::SnapshotStorage;
|
||||
use crate::workspace::WorkspaceMetadata;
|
||||
|
||||
fn workspace_metadata(path: impl Into<PathBuf>) -> WorkspaceMetadata {
|
||||
WorkspaceMetadata {
|
||||
path: path.into(),
|
||||
navigated_ts: None,
|
||||
modified_ts: None,
|
||||
queried_ts: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn codebase_index_status(
|
||||
has_pending: bool,
|
||||
has_synced_version: bool,
|
||||
last_sync_successful: Option<CodebaseIndexFinishedStatus>,
|
||||
sync_progress: Option<SyncProgress>,
|
||||
) -> CodebaseIndexStatus {
|
||||
CodebaseIndexStatus {
|
||||
has_pending,
|
||||
has_synced_version,
|
||||
last_sync_successful,
|
||||
sync_progress,
|
||||
root_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codebase_index_status_event_key_matches_identical_statuses() {
|
||||
let first_status = codebase_index_status(
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
Some(SyncProgress::Syncing {
|
||||
completed_nodes: 1,
|
||||
total_nodes: 2,
|
||||
}),
|
||||
);
|
||||
let duplicate_status = codebase_index_status(
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
Some(SyncProgress::Syncing {
|
||||
completed_nodes: 1,
|
||||
total_nodes: 2,
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
CodebaseIndexStatusEventKey::from(&first_status),
|
||||
CodebaseIndexStatusEventKey::from(&duplicate_status)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codebase_index_status_event_key_detects_semantic_changes() {
|
||||
let syncing_status = codebase_index_status(
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
Some(SyncProgress::Syncing {
|
||||
completed_nodes: 1,
|
||||
total_nodes: 2,
|
||||
}),
|
||||
);
|
||||
let completed_status = codebase_index_status(
|
||||
false,
|
||||
true,
|
||||
Some(CodebaseIndexFinishedStatus::Completed),
|
||||
None,
|
||||
);
|
||||
let failed_status = codebase_index_status(
|
||||
false,
|
||||
true,
|
||||
Some(CodebaseIndexFinishedStatus::Failed(
|
||||
CodebaseIndexingError::BuildTreeError,
|
||||
)),
|
||||
None,
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
CodebaseIndexStatusEventKey::from(&syncing_status),
|
||||
CodebaseIndexStatusEventKey::from(&completed_status)
|
||||
);
|
||||
assert_ne!(
|
||||
CodebaseIndexStatusEventKey::from(&completed_status),
|
||||
CodebaseIndexStatusEventKey::from(&failed_status)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initializes_with_indexing_enabled_when_configured() {
|
||||
App::test((), |app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
vec![workspace_metadata("repo")],
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.read(&app, |manager, _| {
|
||||
assert!(manager.is_indexing_enabled());
|
||||
assert_eq!(manager.num_active_indices(), 0);
|
||||
assert!(manager.can_create_new_indices());
|
||||
});
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn initializes_with_indexing_disabled_when_configured() {
|
||||
App::test((), |app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
vec![workspace_metadata("repo")],
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
false,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.read(&app, |manager, _| {
|
||||
assert!(!manager.is_indexing_enabled());
|
||||
assert_eq!(manager.num_active_indices(), 0);
|
||||
assert!(!manager.can_create_new_indices());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "local_fs")]
|
||||
fn initializes_with_injected_snapshot_storage_when_configured() {
|
||||
App::test((), |app| async move {
|
||||
let snapshot_dir = tempfile::tempdir().unwrap();
|
||||
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
|
||||
let expected_snapshot_dir = storage.path().to_path_buf();
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new_with_snapshot_storage(
|
||||
CodebaseIndexManagerConfig::new(
|
||||
vec![workspace_metadata("repo")],
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
false,
|
||||
),
|
||||
Some(storage),
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.read(&app, |manager, _| {
|
||||
let snapshot_storage = manager.snapshot_storage.as_ref().unwrap();
|
||||
assert_eq!(snapshot_storage.path(), expected_snapshot_dir);
|
||||
assert!(!snapshot_storage.is_app_default());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_index_restore_starts_on_startup_by_default() {
|
||||
App::test((), |app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.read(&app, |manager, _| {
|
||||
assert!(manager.build_queue.is_running());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "local_fs")]
|
||||
fn deferred_persisted_index_restore_starts_once() {
|
||||
App::test((), |mut app| async move {
|
||||
app.add_singleton_model(DirectoryWatcher::new);
|
||||
|
||||
let snapshot_dir = tempfile::tempdir().unwrap();
|
||||
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
|
||||
let first_repo = tempfile::tempdir().unwrap();
|
||||
let second_repo = tempfile::tempdir().unwrap();
|
||||
let mut first_metadata = workspace_metadata(first_repo.path());
|
||||
first_metadata.modified_ts = Some(Utc::now());
|
||||
let mut second_metadata = workspace_metadata(second_repo.path());
|
||||
second_metadata.modified_ts = Some(Utc::now());
|
||||
std::fs::write(storage.snapshot_path(first_repo.path()), b"snapshot").unwrap();
|
||||
std::fs::write(storage.snapshot_path(second_repo.path()), b"snapshot").unwrap();
|
||||
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new_with_snapshot_storage(
|
||||
CodebaseIndexManagerConfig::new(
|
||||
vec![first_metadata, second_metadata],
|
||||
Some(2),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
)
|
||||
.defer_persisted_index_restore(),
|
||||
Some(storage),
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
assert!(!manager.build_queue.is_running());
|
||||
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 2);
|
||||
|
||||
manager.start_persisted_index_restore(ctx);
|
||||
assert!(manager.build_queue.is_running());
|
||||
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 1);
|
||||
|
||||
manager.start_persisted_index_restore(ctx);
|
||||
assert_eq!(manager.build_queue.queued_metadata().into_iter().count(), 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn can_create_new_indices_honors_max_limit_when_enabled() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
assert!(manager.can_create_new_indices());
|
||||
manager.update_max_limits(Some(0), 1000, 32, ctx);
|
||||
assert!(!manager.can_create_new_indices());
|
||||
});
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn index_directory_is_noop_when_indexing_disabled() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
false,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
assert!(!manager.index_directory(PathBuf::from("repo"), ctx));
|
||||
assert_eq!(manager.num_active_indices(), 0);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_directory_reports_when_max_index_limit_prevents_creation() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(0),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
assert!(!manager.index_directory(PathBuf::from("repo"), ctx));
|
||||
assert_eq!(manager.num_active_indices(), 0);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_and_sync_is_noop_when_indexing_disabled() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
false,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
assert!(!manager
|
||||
.build_and_sync_codebase_index(BuildSource::FromPath(Path::new("repo")), ctx));
|
||||
assert_eq!(manager.num_active_indices(), 0);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trigger_incremental_sync_returns_err_when_enabled_and_index_missing() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
true,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
let result = manager.trigger_incremental_sync_for_path(Path::new("repo"), ctx);
|
||||
assert!(result.is_err());
|
||||
});
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn trigger_incremental_sync_returns_ok_when_indexing_disabled() {
|
||||
App::test((), |mut app| async move {
|
||||
let manager = app.add_singleton_model(|ctx| {
|
||||
CodebaseIndexManager::new(
|
||||
Vec::new(),
|
||||
Some(1),
|
||||
1000,
|
||||
32,
|
||||
Arc::new(MockStoreClient),
|
||||
false,
|
||||
ctx,
|
||||
)
|
||||
});
|
||||
|
||||
manager.update(&mut app, |manager, ctx| {
|
||||
let result = manager.trigger_incremental_sync_for_path(Path::new("repo"), ctx);
|
||||
assert!(result.is_ok());
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,13 +1,16 @@
|
||||
//! Common types for hashes that identify codebase embedding state.
|
||||
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use generic_array::GenericArray;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{digest::OutputSizeUser, Digest, Sha256};
|
||||
use std::{fmt, str::FromStr, sync::Arc};
|
||||
|
||||
use crate::index::full_source_code_embedding::chunker::Fragment;
|
||||
use sha2::digest::OutputSizeUser;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::Error;
|
||||
use crate::index::full_source_code_embedding::chunker::Fragment;
|
||||
|
||||
/// The hash of an *intermediate* node in the [`MerkleTree`].
|
||||
///
|
||||
@@ -223,5 +226,5 @@ impl fmt::Display for MerkleHash {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "hash_test.rs"]
|
||||
#[path = "hash_tests.rs"]
|
||||
mod hash_test;
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::MerkleHash;
|
||||
use crate::index::full_source_code_embedding::chunker::Fragment;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[test]
|
||||
fn test_fragment_hash_from_content() {
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::{chunker::Fragment, Error};
|
||||
use super::chunker::Fragment;
|
||||
use super::Error;
|
||||
|
||||
mod hash;
|
||||
mod node;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use crate::index::{
|
||||
THREADPOOL, {DirectoryEntry, Entry, FileMetadata},
|
||||
};
|
||||
use anyhow::anyhow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::ops::Range;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use chrono::{DateTime, Utc};
|
||||
use galaxy_util::standardized_path::StandardizedPath;
|
||||
use itertools::Itertools;
|
||||
@@ -10,26 +10,18 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
use repo_metadata::entry::is_file_parsable;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
use crate::index::full_source_code_embedding::{
|
||||
chunker::chunk_code,
|
||||
fragment_metadata::{FragmentMetadata, LeafToFragmentMetadataUpdates},
|
||||
Error,
|
||||
};
|
||||
|
||||
use super::{
|
||||
hash::MerkleHash,
|
||||
serialized_tree::{SerializedFilesystemInfo, SerializedMerkleNode},
|
||||
tree::UpdateFileResult,
|
||||
ContentHash, DirEntryOrFragment, NodeHash,
|
||||
use super::hash::MerkleHash;
|
||||
use super::serialized_tree::{SerializedFilesystemInfo, SerializedMerkleNode};
|
||||
use super::tree::UpdateFileResult;
|
||||
use super::{ContentHash, DirEntryOrFragment, NodeHash};
|
||||
use crate::index::full_source_code_embedding::chunker::chunk_code;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
FragmentMetadata, LeafToFragmentMetadataUpdates,
|
||||
};
|
||||
use crate::index::full_source_code_embedding::Error;
|
||||
use crate::index::{DirectoryEntry, Entry, FileMetadata, THREADPOOL};
|
||||
|
||||
/// ID that uniquely identifies a node in the merkle tree. It contains the node type
|
||||
/// as well as metadata that distinguishes nodes of the same type.
|
||||
@@ -707,5 +699,5 @@ impl NodeMask {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "node_test.rs"]
|
||||
#[path = "node_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
+4
-5
@@ -1,12 +1,11 @@
|
||||
use crate::index::full_source_code_embedding::{
|
||||
fragment_metadata::LeafToFragmentMetadataUpdates, merkle_tree::DirEntryOrFragment,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
use repo_metadata::{DirectoryEntry, Entry};
|
||||
use virtual_fs::{Stub, VirtualFS};
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::{MerkleNode, NodeMask};
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadataUpdates;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::DirEntryOrFragment;
|
||||
|
||||
/// Tests that node hashes for directories are sorted (meaning they are resilient to files within
|
||||
/// the directory being in a different order).
|
||||
@@ -1,18 +1,17 @@
|
||||
use std::{
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use super::{hash::MerkleHash, node::NodeId, MerkleTree, NodeHash, NodeLens};
|
||||
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
FragmentLocation, LeafToFragmentMetadata,
|
||||
};
|
||||
use std::ops::Range;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
use super::hash::MerkleHash;
|
||||
use super::node::NodeId;
|
||||
use super::{MerkleTree, NodeHash, NodeLens};
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
FragmentLocation, LeafToFragmentMetadata,
|
||||
};
|
||||
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub(crate) struct SerializedCodebaseIndex {
|
||||
tree: SerializedMerkleTree,
|
||||
@@ -196,5 +195,5 @@ impl SerializedMerkleNode {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "serialized_tree_test.rs"]
|
||||
#[path = "serialized_tree_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
+1
-2
@@ -2,12 +2,11 @@ use futures::executor::block_on;
|
||||
use serde_json;
|
||||
use virtual_fs::VirtualFS;
|
||||
|
||||
use super::SerializedCodebaseIndex;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::{
|
||||
construct_test_merkle_tree, MerkleTree,
|
||||
};
|
||||
|
||||
use super::SerializedCodebaseIndex;
|
||||
|
||||
#[test]
|
||||
fn round_trip_index_serialize_deserialize_json() {
|
||||
VirtualFS::test("test_nodes_from_path_json", |dirs, mut sandbox| {
|
||||
@@ -1,8 +1,9 @@
|
||||
use super::MerkleTree;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadata;
|
||||
use repo_metadata::{DirectoryEntry, Entry};
|
||||
use virtual_fs::{Stub, VirtualFS};
|
||||
|
||||
use super::MerkleTree;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadata;
|
||||
|
||||
/// Construct a test Merkle tree with the following structure:
|
||||
/// ```
|
||||
/// root.txt
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
use crate::index::Entry;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use cfg_if::cfg_if;
|
||||
use std::{
|
||||
collections::{HashSet, VecDeque},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use super::node::{ChildrenPath, MerkleNode, NodeLens, NodeMask};
|
||||
use super::serialized_tree::SerializedMerkleTree;
|
||||
use super::DirEntryOrFragment;
|
||||
use crate::index::full_source_code_embedding::fragment_metadata::{
|
||||
LeafToFragmentMetadata, LeafToFragmentMetadataUpdates,
|
||||
};
|
||||
use crate::index::full_source_code_embedding::Error;
|
||||
|
||||
use super::{
|
||||
node::{ChildrenPath, MerkleNode, NodeLens, NodeMask},
|
||||
serialized_tree::SerializedMerkleTree,
|
||||
DirEntryOrFragment,
|
||||
};
|
||||
use crate::index::Entry;
|
||||
|
||||
pub(super) enum UpdateFileResult {
|
||||
Deleted,
|
||||
@@ -210,5 +206,5 @@ impl MerkleTree {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tree_test.rs"]
|
||||
#[path = "tree_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
+2
-4
@@ -1,11 +1,9 @@
|
||||
use crate::index::full_source_code_embedding::merkle_tree::{
|
||||
construct_test_merkle_tree, node::ChildrenPath,
|
||||
};
|
||||
use futures::executor::block_on;
|
||||
|
||||
use virtual_fs::VirtualFS;
|
||||
|
||||
use super::*;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::construct_test_merkle_tree;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::node::ChildrenPath;
|
||||
|
||||
#[test]
|
||||
fn test_nodes_from_path() {
|
||||
@@ -5,19 +5,21 @@ mod fragment_metadata;
|
||||
pub mod manager;
|
||||
mod merkle_tree;
|
||||
mod priority_queue;
|
||||
pub mod search_shaping;
|
||||
mod snapshot;
|
||||
pub mod store_client;
|
||||
mod sync_client;
|
||||
|
||||
use std::{ops::Range, path::PathBuf, time::Duration};
|
||||
pub use sync_client::SyncTask;
|
||||
use std::ops::Range;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
pub use codebase_index::{CodebaseIndex, RetrievalID, SyncProgress};
|
||||
pub use fragment_metadata::{FragmentLocation as FragmentMetadataLocation, FragmentMetadata};
|
||||
pub use merkle_tree::{ContentHash, NodeHash};
|
||||
|
||||
use fragment_metadata::FragmentMetadata;
|
||||
use galaxy_graphql::queries::rerank_fragments::FragmentLocationInput;
|
||||
pub use snapshot::SnapshotStorage;
|
||||
use string_offset::ByteOffset;
|
||||
pub use sync_client::SyncTask;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
@@ -87,6 +89,7 @@ pub enum EmbeddingConfig {
|
||||
Voyage3_5_Lite_512,
|
||||
#[default]
|
||||
Voyage3_5_512,
|
||||
Voyage4_512,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -115,6 +118,9 @@ impl From<EmbeddingConfig> for galaxy_graphql::full_source_code_embedding::Embed
|
||||
EmbeddingConfig::Voyage3_5_Lite_512 => {
|
||||
galaxy_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35Lite512
|
||||
}
|
||||
EmbeddingConfig::Voyage4_512 => {
|
||||
warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage4512
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,6 +144,9 @@ impl TryFrom<galaxy_graphql::full_source_code_embedding::EmbeddingConfig> for Em
|
||||
galaxy_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35512 => {
|
||||
Ok(Self::Voyage3_5_512)
|
||||
}
|
||||
warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage4512 => {
|
||||
Ok(Self::Voyage4_512)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -161,6 +170,36 @@ pub struct Fragment {
|
||||
location: FragmentLocation,
|
||||
}
|
||||
|
||||
impl Fragment {
|
||||
pub fn from_byte_range(
|
||||
content: String,
|
||||
content_hash: ContentHash,
|
||||
absolute_path: PathBuf,
|
||||
byte_range: Range<ByteOffset>,
|
||||
) -> Self {
|
||||
Self {
|
||||
content,
|
||||
content_hash,
|
||||
location: FragmentLocation {
|
||||
absolute_path,
|
||||
byte_range,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn content_hash(&self) -> &ContentHash {
|
||||
&self.content_hash
|
||||
}
|
||||
|
||||
pub fn absolute_path(&self) -> &Path {
|
||||
&self.location.absolute_path
|
||||
}
|
||||
|
||||
pub fn byte_range(&self) -> Range<ByteOffset> {
|
||||
self.location.byte_range.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Fragment> for galaxy_graphql::full_source_code_embedding::Fragment {
|
||||
fn from(val: Fragment) -> Self {
|
||||
Self {
|
||||
|
||||
@@ -18,6 +18,14 @@ struct QueueEntry {
|
||||
metadata: WorkspaceMetadata,
|
||||
}
|
||||
|
||||
/// Controls whether queued builds may be consumed.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
enum BuildQueueState {
|
||||
Paused,
|
||||
#[default]
|
||||
Running,
|
||||
}
|
||||
|
||||
impl Hash for QueueEntry {
|
||||
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
|
||||
self.metadata.path.hash(state);
|
||||
@@ -35,6 +43,7 @@ impl Eq for QueueEntry {}
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct BuildQueue {
|
||||
queue: PriorityQueue<QueueEntry, Priority>,
|
||||
state: BuildQueueState,
|
||||
}
|
||||
|
||||
impl BuildQueue {
|
||||
@@ -46,7 +55,10 @@ impl BuildQueue {
|
||||
self.queue.iter().map(|(entry, _)| entry.metadata.clone())
|
||||
}
|
||||
|
||||
pub(super) fn new_with_persisted(snapshots_to_load: Vec<WorkspaceMetadata>) -> Self {
|
||||
pub(super) fn new_with_persisted(
|
||||
snapshots_to_load: Vec<WorkspaceMetadata>,
|
||||
start_immediately: bool,
|
||||
) -> Self {
|
||||
let mut queue = PriorityQueue::new();
|
||||
queue.extend(
|
||||
snapshots_to_load
|
||||
@@ -54,12 +66,35 @@ impl BuildQueue {
|
||||
.sorted_by(WorkspaceMetadata::most_recently_touched)
|
||||
.map(|entry| (QueueEntry { metadata: entry }, Priority::PersistedSnapshot)),
|
||||
);
|
||||
let state = if start_immediately {
|
||||
BuildQueueState::Running
|
||||
} else {
|
||||
BuildQueueState::Paused
|
||||
};
|
||||
|
||||
Self { queue }
|
||||
Self { queue, state }
|
||||
}
|
||||
|
||||
pub(super) fn is_running(&self) -> bool {
|
||||
self.state == BuildQueueState::Running
|
||||
}
|
||||
|
||||
/// Starts consuming queued builds. Returns whether the queue transitioned to running.
|
||||
pub(super) fn start(&mut self) -> bool {
|
||||
match self.state {
|
||||
BuildQueueState::Paused => {
|
||||
self.state = BuildQueueState::Running;
|
||||
true
|
||||
}
|
||||
BuildQueueState::Running => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pulls the next index root path to sync from the priority queue and returns it.
|
||||
pub fn pick_next_sync(&mut self) -> Option<WorkspaceMetadata> {
|
||||
if !self.is_running() {
|
||||
return None;
|
||||
}
|
||||
self.queue.pop().map(|(entry, _priority)| entry.metadata)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::{ContentHash, Fragment, FragmentLocation, FragmentMetadata};
|
||||
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ReadFragmentResult {
|
||||
pub successfully_read: Vec<Fragment>,
|
||||
pub fail_to_read: Vec<ContentHash>,
|
||||
pub fail_to_read_path: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
pub fn build_fragments_from_file_contents(
|
||||
metadatas: impl IntoIterator<Item = (ContentHash, FragmentMetadata)>,
|
||||
file_contents: &HashMap<PathBuf, String>,
|
||||
) -> ReadFragmentResult {
|
||||
let mut fragments = Vec::new();
|
||||
let mut fail_to_read = Vec::new();
|
||||
let mut fail_to_read_path = Vec::new();
|
||||
|
||||
// Group fragments by file path.
|
||||
let mut fragments_by_path: HashMap<_, Vec<_>> = HashMap::new();
|
||||
for (content_hash, metadata) in metadatas {
|
||||
fragments_by_path
|
||||
.entry(metadata.absolute_path)
|
||||
.or_default()
|
||||
.push((content_hash, metadata.location.byte_range));
|
||||
}
|
||||
|
||||
// Process each file and its fragments.
|
||||
for (file_path, file_fragments) in fragments_by_path {
|
||||
let mut has_failed_to_read_fragments = false;
|
||||
if let Some(file_content) = file_contents.get(&file_path) {
|
||||
// Process all fragments for this file.
|
||||
for (content_hash, fragment_ranges) in file_fragments {
|
||||
let start_idx = fragment_ranges.start.as_usize();
|
||||
let end_idx = fragment_ranges.end.as_usize();
|
||||
|
||||
if start_idx <= end_idx
|
||||
&& end_idx <= file_content.len()
|
||||
&& file_content.is_char_boundary(start_idx)
|
||||
&& file_content.is_char_boundary(end_idx)
|
||||
{
|
||||
let content = file_content[start_idx..end_idx].to_string();
|
||||
if content.is_empty() {
|
||||
log::trace!(
|
||||
"Fragment for {:?} with range {:?} is empty",
|
||||
file_path.display(),
|
||||
fragment_ranges
|
||||
);
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
} else if ContentHash::from_content(&content) != content_hash {
|
||||
log::trace!(
|
||||
"Fragment for {:?} with range {:?} does not match its content hash",
|
||||
file_path.display(),
|
||||
fragment_ranges
|
||||
);
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
} else {
|
||||
fragments.push(Fragment {
|
||||
content,
|
||||
content_hash,
|
||||
location: FragmentLocation {
|
||||
absolute_path: file_path.clone(),
|
||||
byte_range: fragment_ranges,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else {
|
||||
log::trace!("Invalid byte range {fragment_ranges:?} for file: {file_path:?}");
|
||||
fail_to_read.push(content_hash);
|
||||
has_failed_to_read_fragments = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log::trace!("Failed to read file: {file_path:?}");
|
||||
fail_to_read.extend(
|
||||
file_fragments
|
||||
.into_iter()
|
||||
.map(|(content_hash, _)| content_hash),
|
||||
);
|
||||
has_failed_to_read_fragments = true;
|
||||
}
|
||||
|
||||
if has_failed_to_read_fragments {
|
||||
fail_to_read_path.push(file_path);
|
||||
}
|
||||
}
|
||||
|
||||
ReadFragmentResult {
|
||||
successfully_read: fragments,
|
||||
fail_to_read,
|
||||
fail_to_read_path,
|
||||
}
|
||||
}
|
||||
|
||||
// Convert fragments into CodeContextLocations. This function groups and dedupes fragments in the same file.
|
||||
// It also allows the caller to define a context line number surrounding the relevant fragment.
|
||||
pub fn fragments_to_context_locations<'a>(
|
||||
fragments: Vec<Fragment>,
|
||||
metadata_for_hash: impl Fn(&ContentHash) -> Option<&'a [FragmentMetadata]>,
|
||||
context_lines: usize,
|
||||
) -> HashSet<CodeContextLocation> {
|
||||
// Map to collect fragments by file path.
|
||||
let mut fragments_by_path: HashMap<&PathBuf, Vec<Range<usize>>> = HashMap::new();
|
||||
let mut whole_files = HashSet::new();
|
||||
|
||||
// First pass - collect all fragments and their line ranges by file path.
|
||||
for fragment in &fragments {
|
||||
if let Some(metadata) = metadata_for_hash(&fragment.content_hash).and_then(|metadatas| {
|
||||
metadatas.iter().find(|m| {
|
||||
m.absolute_path == fragment.location.absolute_path
|
||||
&& m.location.byte_range == fragment.location.byte_range
|
||||
})
|
||||
}) {
|
||||
// Add line range with context to the appropriate file's collection.
|
||||
let path = &fragment.location.absolute_path;
|
||||
let start = metadata.location.start_line.saturating_sub(context_lines);
|
||||
let end = metadata.location.end_line + 1 + context_lines;
|
||||
|
||||
fragments_by_path.entry(path).or_default().push(start..end);
|
||||
} else {
|
||||
// Fallback to whole file if metadata not found.
|
||||
whole_files.insert(fragment.location.absolute_path.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass - process each file's fragments.
|
||||
let mut result = HashSet::new();
|
||||
|
||||
// Process each file's fragments.
|
||||
for (path, mut line_ranges) in fragments_by_path {
|
||||
if line_ranges.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// We can skip the fragments if the entire file is already included in the context.
|
||||
if whole_files.contains(path) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sort ranges by start position.
|
||||
line_ranges.sort_by_key(|range| range.start);
|
||||
|
||||
// Merge overlapping or adjacent ranges.
|
||||
let mut merged_ranges: Vec<Range<usize>> = Vec::new();
|
||||
for range in line_ranges {
|
||||
if let Some(last) = merged_ranges.last_mut() {
|
||||
// If current range overlaps or is adjacent to the last one, merge them.
|
||||
if range.start <= last.end {
|
||||
last.end = last.end.max(range.end);
|
||||
} else {
|
||||
merged_ranges.push(range);
|
||||
}
|
||||
} else {
|
||||
merged_ranges.push(range);
|
||||
}
|
||||
}
|
||||
|
||||
// Add file fragment location with all merged ranges.
|
||||
result.insert(CodeContextLocation::Fragment(FileFragmentLocation {
|
||||
path: path.clone(),
|
||||
line_ranges: merged_ranges,
|
||||
}));
|
||||
}
|
||||
|
||||
// Add whole files to the result set.
|
||||
result.extend(whole_files.into_iter().map(CodeContextLocation::WholeFile));
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "search_shaping_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,133 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
use super::super::{ContentHash, Fragment, FragmentLocation, FragmentMetadata};
|
||||
use super::{build_fragments_from_file_contents, fragments_to_context_locations};
|
||||
use crate::index::locations::{CodeContextLocation, FileFragmentLocation};
|
||||
|
||||
fn metadata(
|
||||
path: &str,
|
||||
byte_range: Range<ByteOffset>,
|
||||
start_line: usize,
|
||||
end_line: usize,
|
||||
) -> FragmentMetadata {
|
||||
FragmentMetadata {
|
||||
absolute_path: PathBuf::from(path),
|
||||
location: super::super::fragment_metadata::FragmentLocation {
|
||||
start_line,
|
||||
end_line,
|
||||
byte_range,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn fragment(content: &str, path: &str, byte_range: Range<ByteOffset>) -> Fragment {
|
||||
Fragment {
|
||||
content: content.to_string(),
|
||||
content_hash: ContentHash::from_content(content),
|
||||
location: FragmentLocation {
|
||||
absolute_path: PathBuf::from(path),
|
||||
byte_range,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_fragments_from_exact_byte_ranges() {
|
||||
let path = PathBuf::from("/repo/src/lib.rs");
|
||||
let content = "before\nneedle\nπ-after".to_string();
|
||||
let fragment_content = "needle";
|
||||
let start = content.find(fragment_content).unwrap();
|
||||
let end = start + fragment_content.len();
|
||||
let content_hash = ContentHash::from_content(fragment_content);
|
||||
let metadata = metadata(
|
||||
path.to_string_lossy().as_ref(),
|
||||
ByteOffset::from(start)..ByteOffset::from(end),
|
||||
2,
|
||||
2,
|
||||
);
|
||||
|
||||
let result = build_fragments_from_file_contents(
|
||||
[(content_hash.clone(), metadata)],
|
||||
&HashMap::from([(path.clone(), content)]),
|
||||
);
|
||||
|
||||
assert_eq!(result.fail_to_read.len(), 0);
|
||||
assert_eq!(result.successfully_read.len(), 1);
|
||||
let fragment = &result.successfully_read[0];
|
||||
assert_eq!(fragment.content, fragment_content);
|
||||
assert_eq!(fragment.content_hash, content_hash);
|
||||
assert_eq!(fragment.location.absolute_path, path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_hashes_and_byte_ranges() {
|
||||
let path = PathBuf::from("/repo/src/lib.rs");
|
||||
let content = "abcπdef".to_string();
|
||||
let bad_hash_metadata = metadata(
|
||||
path.to_string_lossy().as_ref(),
|
||||
ByteOffset::from(0)..ByteOffset::from(3),
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let invalid_boundary_metadata = metadata(
|
||||
path.to_string_lossy().as_ref(),
|
||||
ByteOffset::from(4)..ByteOffset::from(5),
|
||||
1,
|
||||
1,
|
||||
);
|
||||
|
||||
let result = build_fragments_from_file_contents(
|
||||
[
|
||||
(ContentHash::from_content("not abc"), bad_hash_metadata),
|
||||
(ContentHash::from_content("π"), invalid_boundary_metadata),
|
||||
],
|
||||
&HashMap::from([(path.clone(), content)]),
|
||||
);
|
||||
|
||||
assert!(result.successfully_read.is_empty());
|
||||
assert_eq!(result.fail_to_read.len(), 2);
|
||||
assert_eq!(result.fail_to_read_path, vec![path]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shapes_fragments_into_merged_context_locations() {
|
||||
let path = "/repo/src/lib.rs";
|
||||
let fragment_a = fragment("a", path, ByteOffset::from(0)..ByteOffset::from(1));
|
||||
let fragment_b = fragment("b", path, ByteOffset::from(2)..ByteOffset::from(3));
|
||||
let metadata_a = metadata(path, ByteOffset::from(0)..ByteOffset::from(1), 10, 12);
|
||||
let metadata_b = metadata(path, ByteOffset::from(2)..ByteOffset::from(3), 15, 17);
|
||||
let metadata_by_hash = HashMap::from([
|
||||
(fragment_a.content_hash.clone(), vec![metadata_a]),
|
||||
(fragment_b.content_hash.clone(), vec![metadata_b]),
|
||||
]);
|
||||
|
||||
let result = fragments_to_context_locations(
|
||||
vec![fragment_a, fragment_b],
|
||||
|hash| metadata_by_hash.get(hash).map(Vec::as_slice),
|
||||
2,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
HashSet::from([CodeContextLocation::Fragment(FileFragmentLocation {
|
||||
path: PathBuf::from(path),
|
||||
line_ranges: std::iter::once(8..20).collect(),
|
||||
})])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_whole_file_when_metadata_is_missing() {
|
||||
let path = "/repo/src/lib.rs";
|
||||
let fragment = fragment("a", path, ByteOffset::from(0)..ByteOffset::from(1));
|
||||
let result = fragments_to_context_locations(vec![fragment], |_| None, 2);
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
HashSet::from([CodeContextLocation::WholeFile(PathBuf::from(path))])
|
||||
);
|
||||
}
|
||||
@@ -1,20 +1,21 @@
|
||||
use std::collections::HashSet;
|
||||
use std::hash::{DefaultHasher, Hash, Hasher};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
#[cfg(feature = "local_fs")]
|
||||
use galaxyui::ModelHandle;
|
||||
#[cfg(feature = "local_fs")]
|
||||
use repo_metadata::Repository;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
#[cfg(feature = "local_fs")]
|
||||
use galaxyui_core::ModelHandle;
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
use super::Error as CodebaseIndexError;
|
||||
use std::sync::Arc;
|
||||
use galaxyui::ModelContext;
|
||||
use galaxyui_core::ModelContext;
|
||||
use anyhow::Context;
|
||||
use galaxy_core::safe_info;
|
||||
use super::{store_client::StoreClient, CodebaseIndex, EmbeddingConfig};
|
||||
@@ -33,11 +34,47 @@ const REPO_SNAPSHOT_SHELF_LIFE_DURATION: Duration =
|
||||
|
||||
/// Subdirectory inside the app's statedirectory that holds snapshot files.
|
||||
const REPO_SNAPSHOT_SUBDIR_NAME: &str = "codebase_index_snapshots";
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct SnapshotStorage {
|
||||
dir: PathBuf,
|
||||
}
|
||||
|
||||
impl SnapshotStorage {
|
||||
/// Construct snapshot storage using the app's default secure snapshot directory.
|
||||
pub fn app_default() -> Option<Self> {
|
||||
snapshot_dir().map(|dir| Self { dir })
|
||||
}
|
||||
|
||||
/// Construct snapshot storage rooted at the supplied directory, creating it if needed.
|
||||
pub fn from_dir(dir: PathBuf) -> Option<Self> {
|
||||
if !dir.is_dir() {
|
||||
std::fs::create_dir_all(&dir).ok()?;
|
||||
}
|
||||
Some(Self { dir })
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
#[cfg(feature = "local_fs")]
|
||||
pub(super) fn is_app_default(&self) -> bool {
|
||||
self.dir == default_snapshot_dir_path()
|
||||
}
|
||||
|
||||
pub(super) fn has_snapshot(&self, repo_path: &Path) -> bool {
|
||||
self.snapshot_path(repo_path).is_file()
|
||||
}
|
||||
|
||||
pub(super) fn snapshot_path(&self, repo_path: &Path) -> PathBuf {
|
||||
snapshot_path(&self.dir, repo_path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Splits a list of codebase indices into invalid and valid indices,
|
||||
/// based on their last write date and whether they have a corresponding snapshot file.
|
||||
pub(super) fn split_snapshot_metadata_by_validity(
|
||||
persisted_codebase_indices: Vec<WorkspaceMetadata>,
|
||||
snapshot_storage: Option<&SnapshotStorage>,
|
||||
) -> (Vec<WorkspaceMetadata>, Vec<WorkspaceMetadata>) {
|
||||
let now = Utc::now();
|
||||
persisted_codebase_indices
|
||||
@@ -48,7 +85,8 @@ pub(super) fn split_snapshot_metadata_by_validity(
|
||||
index_metadata.path
|
||||
);
|
||||
index_metadata.is_expired(now, REPO_SNAPSHOT_SHELF_LIFE_DAYS)
|
||||
|| !has_snapshot(&index_metadata.path)
|
||||
|| !snapshot_storage
|
||||
.is_some_and(|storage| storage.has_snapshot(&index_metadata.path))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -159,12 +197,7 @@ pub(super) fn read_snapshot(
|
||||
}
|
||||
|
||||
pub(super) fn has_snapshot(repo_path: &Path) -> bool {
|
||||
let Some(snapshot_dir) = snapshot_dir() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let snapshot_path = snapshot_path(snapshot_dir.as_path(), repo_path);
|
||||
snapshot_path.is_file()
|
||||
SnapshotStorage::app_default().is_some_and(|storage| storage.has_snapshot(repo_path))
|
||||
}
|
||||
|
||||
/// Construct a directory to store index snapshots, if it doesn't already exist,
|
||||
@@ -175,9 +208,7 @@ pub(super) fn snapshot_dir() -> Option<PathBuf> {
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
{
|
||||
let base_dir =
|
||||
galaxy_core::paths::secure_state_dir().unwrap_or_else(galaxy_core::paths::state_dir);
|
||||
let snapshot_dir_path = base_dir.join(REPO_SNAPSHOT_SUBDIR_NAME);
|
||||
let snapshot_dir_path = default_snapshot_dir_path();
|
||||
|
||||
if !snapshot_dir_path.is_dir() {
|
||||
std::fs::create_dir_all(&snapshot_dir_path).ok()?;
|
||||
@@ -186,9 +217,15 @@ pub(super) fn snapshot_dir() -> Option<PathBuf> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
fn default_snapshot_dir_path() -> PathBuf {
|
||||
galaxy_core::paths::secure_state_dir()
|
||||
.unwrap_or_else(galaxy_core::paths::state_dir)
|
||||
.join(REPO_SNAPSHOT_SUBDIR_NAME)
|
||||
}
|
||||
/// Constructs a snapshot path given a base directory and the codebase index's root path.
|
||||
pub(super) fn snapshot_path(snapshot_dir: &Path, repo_path: &Path) -> PathBuf {
|
||||
// Use a hash the repo_path to create a unique filename
|
||||
// Use a hash of the repo_path to create a unique filename
|
||||
let mut hasher = DefaultHasher::new();
|
||||
repo_path.hash(&mut hasher);
|
||||
let snapshot_file_name = format!("snapshot_{}", hasher.finish());
|
||||
|
||||
@@ -1,7 +1,57 @@
|
||||
use chrono::Duration;
|
||||
use chrono::{Duration, Utc};
|
||||
use virtual_fs::{Stub, VirtualFS};
|
||||
|
||||
use super::*;
|
||||
fn workspace_metadata(path: impl Into<PathBuf>) -> WorkspaceMetadata {
|
||||
WorkspaceMetadata {
|
||||
path: path.into(),
|
||||
navigated_ts: None,
|
||||
modified_ts: Some(Utc::now()),
|
||||
queried_ts: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "local_fs")]
|
||||
fn snapshot_storage_app_default_matches_snapshot_dir() {
|
||||
VirtualFS::test(
|
||||
"snapshot_storage_app_default_matches_snapshot_dir",
|
||||
|_dirs, _sandbox| {
|
||||
let storage = SnapshotStorage::app_default().unwrap();
|
||||
assert_eq!(storage.path(), snapshot_dir().unwrap());
|
||||
assert!(storage.is_app_default());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_snapshot_metadata_by_validity_uses_injected_snapshot_dir() {
|
||||
let snapshot_dir = tempfile::tempdir().unwrap();
|
||||
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
|
||||
let repo_path = PathBuf::from("/remote/repo");
|
||||
std::fs::write(storage.snapshot_path(&repo_path), b"snapshot").unwrap();
|
||||
|
||||
let (invalid_metadata, valid_metadata) =
|
||||
split_snapshot_metadata_by_validity(vec![workspace_metadata(&repo_path)], Some(&storage));
|
||||
|
||||
assert!(invalid_metadata.is_empty());
|
||||
assert_eq!(valid_metadata.len(), 1);
|
||||
assert_eq!(valid_metadata[0].path, repo_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_snapshot_metadata_by_validity_rejects_missing_injected_snapshot() {
|
||||
let snapshot_dir = tempfile::tempdir().unwrap();
|
||||
let storage = SnapshotStorage::from_dir(snapshot_dir.path().join("daemon")).unwrap();
|
||||
let repo_path = PathBuf::from("/remote/repo");
|
||||
|
||||
let (invalid_metadata, valid_metadata) =
|
||||
split_snapshot_metadata_by_validity(vec![workspace_metadata(&repo_path)], Some(&storage));
|
||||
|
||||
assert_eq!(invalid_metadata.len(), 1);
|
||||
assert_eq!(invalid_metadata[0].path, repo_path);
|
||||
assert!(valid_metadata.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_up_snapshot_files() {
|
||||
@@ -71,12 +121,7 @@ fn test_clean_up_snapshot_files() {
|
||||
.unwrap();
|
||||
|
||||
// Create test metadata that only includes the valid file
|
||||
let metadata = vec![WorkspaceMetadata {
|
||||
path: test_path,
|
||||
navigated_ts: None,
|
||||
modified_ts: None,
|
||||
queried_ts: None,
|
||||
}];
|
||||
let metadata = vec![workspace_metadata(test_path)];
|
||||
|
||||
// Run cleanup
|
||||
clean_up_snapshot_files(&snapshot_dir_absolute_path, &metadata);
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Debug;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fmt::Debug,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use super::{
|
||||
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
|
||||
|
||||
@@ -1,28 +1,23 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use galaxy_core::sync_queue::{IsTransientError, SyncQueue, SyncQueueTaskTrait};
|
||||
use itertools::Itertools;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::mem;
|
||||
use std::ops::AddAssign;
|
||||
use std::pin::Pin;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
mem,
|
||||
sync::Arc,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{CodebaseContextConfig, NodeHash};
|
||||
|
||||
use crate::index::full_source_code_embedding::store_client::IntermediateNode;
|
||||
use anyhow::{anyhow, Result};
|
||||
use itertools::Itertools;
|
||||
use galaxy_core::sync_queue::{IsTransientError, SyncQueue, SyncQueueTaskTrait};
|
||||
|
||||
use super::changed_files::ChangedFiles;
|
||||
use super::codebase_index::{build_fragments_from_metadata, SyncProgress};
|
||||
use super::fragment_metadata::LeafToFragmentMetadataMapping;
|
||||
use super::merkle_tree::{MerkleTree, NodeLens};
|
||||
use super::store_client::StoreClient;
|
||||
use super::{
|
||||
changed_files::ChangedFiles,
|
||||
codebase_index::{build_fragments_from_metadata, SyncProgress},
|
||||
fragment_metadata::LeafToFragmentMetadataMapping,
|
||||
merkle_tree::{MerkleTree, NodeLens},
|
||||
store_client::StoreClient,
|
||||
EmbeddingConfig, Error, RepoMetadata,
|
||||
CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
|
||||
};
|
||||
use super::{ContentHash, Fragment};
|
||||
use crate::index::full_source_code_embedding::store_client::IntermediateNode;
|
||||
|
||||
const SYNC_NODE_BATCH_SIZE: usize = 500;
|
||||
// Minimum node batch size used for updates.
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use futures::executor::block_on;
|
||||
use virtual_fs::VirtualFS;
|
||||
|
||||
use super::batch_leaves_by_size;
|
||||
use crate::index::full_source_code_embedding::merkle_tree::{construct_test_merkle_tree, NodeLens};
|
||||
|
||||
use virtual_fs::VirtualFS;
|
||||
|
||||
/// Collect all leaf nodes from a merkle tree by walking it recursively.
|
||||
fn collect_leaves<'a>(node: NodeLens<'a>) -> Vec<NodeLens<'a>> {
|
||||
if node.is_leaf() {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{ops::Range, path::PathBuf};
|
||||
|
||||
/// A line-based file fragment location.
|
||||
///
|
||||
|
||||
@@ -7,15 +7,13 @@ pub mod full_source_code_embedding;
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
pub use file_outline::build_outline;
|
||||
|
||||
pub use file_outline::{Outline, Symbol};
|
||||
pub use repo_metadata::{BuildTreeError, DirectoryEntry, Entry, FileId, FileMetadata};
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
pub use repo_metadata::{
|
||||
matches_gitignores, path_passes_filters,
|
||||
};
|
||||
pub use repo_metadata::entry::{is_git_internal_path, should_watch_directory_in_git_path};
|
||||
pub use repo_metadata::matches_gitignores;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
pub mod agent;
|
||||
pub mod api_keys;
|
||||
pub mod aws_credentials;
|
||||
pub mod geap_credentials;
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub mod grok_subscription;
|
||||
pub mod llm_id;
|
||||
|
||||
pub use llm_id::LLMId;
|
||||
|
||||
@@ -2,8 +2,7 @@ use galaxy_terminal::shell::ShellLaunchData;
|
||||
use galaxy_util::path::{
|
||||
convert_msys2_to_windows_native_path, convert_wsl_to_windows_host_path, msys2_exe_to_root,
|
||||
};
|
||||
use galaxyui::platform::OperatingSystem;
|
||||
use typed_path::{TypedPath, TypedPathBuf, WindowsPath};
|
||||
use galaxyui_core::platform::OperatingSystem;
|
||||
|
||||
fn use_unix_paths(shell: Option<&ShellLaunchData>) -> bool {
|
||||
OperatingSystem::get().is_linux()
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use galaxyui_core::ModelContext;
|
||||
|
||||
use super::model::{ProjectContextModel, ProjectRule};
|
||||
|
||||
/// No-op stand-in for non-`local_fs` builds. File-based global rules require
|
||||
/// filesystem watchers that don't exist on WASM, so callers see an empty
|
||||
/// view here.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct GlobalRules;
|
||||
|
||||
impl GlobalRules {
|
||||
pub(crate) fn index(&mut self, _ctx: &mut ModelContext<ProjectContextModel>) {}
|
||||
pub(crate) fn active_rules(&self) -> impl Iterator<Item = ProjectRule> + '_ {
|
||||
std::iter::empty()
|
||||
}
|
||||
|
||||
pub(crate) fn paths(&self) -> impl Iterator<Item = LocalOrRemotePath> + '_ {
|
||||
std::iter::empty()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use async_channel::Sender;
|
||||
use repo_metadata::repository::{RepositorySubscriber, SubscriberId};
|
||||
use repo_metadata::{DirectoryWatcher, Repository, RepositoryUpdate};
|
||||
use strum::IntoEnumIterator;
|
||||
use strum_macros::EnumIter;
|
||||
use galaxy_core::safe_warn;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
use galaxyui_core::{ModelContext, ModelHandle, SingletonEntity};
|
||||
use watcher::{HomeDirectoryWatcher, HomeDirectoryWatcherEvent};
|
||||
|
||||
use super::model::{GlobalRulesDelta, ProjectContextModel, ProjectContextModelEvent, ProjectRule};
|
||||
|
||||
/// A well-known location under `$HOME` that may contain a global rule file.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)]
|
||||
enum GlobalRuleSource {
|
||||
/// `~/.agents/AGENTS.md`.
|
||||
Agents,
|
||||
}
|
||||
|
||||
impl GlobalRuleSource {
|
||||
/// Display name (used in safe logs that don't expose user paths).
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Agents => "agents",
|
||||
}
|
||||
}
|
||||
|
||||
/// Subdirectory under `$HOME`, e.g. `".agents"`.
|
||||
fn home_subdir(self) -> &'static str {
|
||||
match self {
|
||||
Self::Agents => ".agents",
|
||||
}
|
||||
}
|
||||
|
||||
/// File name within the subdir, e.g. `"AGENTS.md"`.
|
||||
fn file_pattern(self) -> &'static str {
|
||||
match self {
|
||||
Self::Agents => "AGENTS.md",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GlobalSourceWatcherState {
|
||||
repository: ModelHandle<Repository>,
|
||||
subscriber_id: SubscriberId,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GlobalRulesUpdate {
|
||||
/// The [`GlobalRuleSource`] variant that produced this update. The
|
||||
/// receiver uses it to look up the matching `home_subdir`/`file_pattern`
|
||||
/// without needing a per-source channel.
|
||||
source: GlobalRuleSource,
|
||||
update: RepositoryUpdate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct GlobalRules {
|
||||
/// Global rule files keyed by absolute file path. Populated from
|
||||
/// [`GlobalRuleSource`]. Independent of project-level rule indexing.
|
||||
/// Stored in a `BTreeMap` so iteration order is deterministic.
|
||||
pub(super) rules: BTreeMap<PathBuf, ProjectRule>,
|
||||
/// Active home-subdir directory watchers, keyed by the absolute subdir
|
||||
/// path (e.g. `~/.agents`).
|
||||
source_watchers: HashMap<PathBuf, GlobalSourceWatcherState>,
|
||||
/// Sender used by global-rule directory subscribers to push updates back
|
||||
/// into the model's main-thread stream handler.
|
||||
updates_tx: Option<Sender<GlobalRulesUpdate>>,
|
||||
}
|
||||
|
||||
impl GlobalRules {
|
||||
pub(crate) fn active_rules(&self) -> impl Iterator<Item = ProjectRule> + '_ {
|
||||
self.rules.values().cloned()
|
||||
}
|
||||
|
||||
pub(crate) fn paths(&self) -> impl Iterator<Item = LocalOrRemotePath> + '_ {
|
||||
self.rules.keys().cloned().map(LocalOrRemotePath::Local)
|
||||
}
|
||||
|
||||
/// Index all configured global rule sources (see [`GlobalRuleSource`]).
|
||||
///
|
||||
/// All disk I/O is dispatched through `ctx.spawn` so this method does not
|
||||
/// block startup. Subscribes to [`HomeDirectoryWatcher`] to react to
|
||||
/// creation/deletion of the home subdirs at runtime, and registers a
|
||||
/// [`DirectoryWatcher`] per existing subdir for incremental updates.
|
||||
///
|
||||
/// Idempotent: subsequent calls are a no-op once the channel is initialized.
|
||||
pub(crate) fn index(&mut self, ctx: &mut ModelContext<ProjectContextModel>) {
|
||||
if self.updates_tx.is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(home_dir) = dirs::home_dir() else {
|
||||
log::debug!("Home directory not found; skipping global rules indexing");
|
||||
return;
|
||||
};
|
||||
|
||||
// Set up the channel that all per-source subscribers push into.
|
||||
let (tx, rx) = async_channel::unbounded::<GlobalRulesUpdate>();
|
||||
self.updates_tx = Some(tx);
|
||||
|
||||
ctx.spawn_stream_local(
|
||||
rx,
|
||||
|me, update, ctx| {
|
||||
me.global_rules
|
||||
.handle_global_rules_update(update.source, update.update, ctx);
|
||||
},
|
||||
|_, _| {},
|
||||
);
|
||||
|
||||
// React to creation/deletion of home subdirs at runtime.
|
||||
ctx.subscribe_to_model(&HomeDirectoryWatcher::handle(ctx), |me, _, event, ctx| {
|
||||
me.global_rules
|
||||
.handle_home_dir_event_for_global_rules(event, ctx);
|
||||
});
|
||||
|
||||
for source in GlobalRuleSource::iter() {
|
||||
let subdir_path = home_dir.join(source.home_subdir());
|
||||
let target_file = subdir_path.join(source.file_pattern());
|
||||
|
||||
// Initial async read; if the file doesn't exist yet, the watcher
|
||||
// will pick it up on creation.
|
||||
Self::spawn_global_rule_read(target_file, ctx);
|
||||
|
||||
if subdir_path.exists() {
|
||||
self.register_global_source_watcher(source, &subdir_path, ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Async read of a single global rule file. The async block runs on a
|
||||
/// background executor; the main-thread callback updates model state once
|
||||
/// the read completes.
|
||||
fn spawn_global_rule_read(file_path: PathBuf, ctx: &mut ModelContext<ProjectContextModel>) {
|
||||
ctx.spawn(
|
||||
async move {
|
||||
// `read_to_string` returning `Err` (e.g. NotFound, permission
|
||||
// denied, file replaced with a non-regular file) is converted
|
||||
// to `None`; the callback below decides whether that means
|
||||
// "insert/refresh" or "drop a previously-known entry."
|
||||
let content = async_fs::read_to_string(&file_path).await.ok();
|
||||
(file_path, content)
|
||||
},
|
||||
move |me, (file_path, content_opt), ctx| match content_opt {
|
||||
Some(content) => {
|
||||
// Read succeeded: insert (or replace) the rule and notify
|
||||
// subscribers.
|
||||
me.global_rules.rules.insert(
|
||||
file_path.clone(),
|
||||
ProjectRule {
|
||||
// Global rule sources are watched under the local home directory.
|
||||
path: LocalOrRemotePath::Local(file_path.clone()),
|
||||
content,
|
||||
},
|
||||
);
|
||||
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
|
||||
GlobalRulesDelta {
|
||||
discovered_rules: vec![file_path],
|
||||
deleted_rules: vec![],
|
||||
},
|
||||
));
|
||||
}
|
||||
None => {
|
||||
// Drop cached content if file is now unreadable; no-op if it never existed.
|
||||
if me.global_rules.rules.remove(&file_path).is_some() {
|
||||
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
|
||||
GlobalRulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![file_path],
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Register a `DirectoryWatcher` on the given home subdir for incremental
|
||||
/// updates. Idempotent: subsequent calls for an already-watched subdir are
|
||||
/// a no-op (the `subdir_path` key dedups by directory rather than by
|
||||
/// source, so multiple sources sharing a `home_subdir` would only register
|
||||
/// the watcher once — a future change can fan out to multiple file
|
||||
/// patterns by extending the value).
|
||||
///
|
||||
/// The subdir must exist on disk before this is called.
|
||||
/// `DirectoryWatcher::add_directory` rejects non-existent paths, and
|
||||
/// runtime creation is handled by `handle_home_dir_event_for_global_rules`,
|
||||
/// which calls back here once the subdir appears.
|
||||
fn register_global_source_watcher(
|
||||
&mut self,
|
||||
source: GlobalRuleSource,
|
||||
subdir_path: &Path,
|
||||
ctx: &mut ModelContext<ProjectContextModel>,
|
||||
) {
|
||||
// If the subdir is already being watched, return early.
|
||||
if self.source_watchers.contains_key(subdir_path) {
|
||||
return;
|
||||
}
|
||||
|
||||
let (Some(update_tx), Ok(std_path)) = (
|
||||
self.updates_tx.clone(),
|
||||
StandardizedPath::from_local_canonicalized(subdir_path),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let repo_handle = match DirectoryWatcher::handle(ctx)
|
||||
.update(ctx, |watcher, ctx| watcher.add_directory(std_path, ctx))
|
||||
{
|
||||
Ok(handle) => handle,
|
||||
Err(err) => {
|
||||
// `safe_warn!` because the path contains the user's home dir,
|
||||
// which is PII; we only want the full path on dogfood builds.
|
||||
// The error itself can also embed the canonicalized path
|
||||
// (e.g. `RepoMetadataError::RepoNotFound(...)`), so we keep
|
||||
// it out of the safe branch as well — only the source name
|
||||
// is safe to send to Sentry.
|
||||
safe_warn!(
|
||||
safe: (
|
||||
"Failed to register {} for global rules watching",
|
||||
source.name()
|
||||
),
|
||||
full: (
|
||||
"Failed to register {} for global rules watching: {err}",
|
||||
subdir_path.display()
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let subscriber = Box::new(GlobalRulesRepositorySubscriber { source, update_tx });
|
||||
|
||||
let start = repo_handle.update(ctx, |repo, ctx| repo.start_watching(subscriber, ctx));
|
||||
let subscriber_id = start.subscriber_id;
|
||||
let subdir_path_owned = subdir_path.to_path_buf();
|
||||
|
||||
self.source_watchers.insert(
|
||||
subdir_path_owned.clone(),
|
||||
GlobalSourceWatcherState {
|
||||
repository: repo_handle.clone(),
|
||||
subscriber_id,
|
||||
},
|
||||
);
|
||||
|
||||
let cleanup_key = subdir_path_owned.clone();
|
||||
let subdir_for_log = subdir_path_owned;
|
||||
ctx.spawn(start.registration_future, move |me, res, ctx| {
|
||||
if let Err(err) = res {
|
||||
// Same PII shape as the registration error above: the path
|
||||
// and the error can both contain the user's home dir, so
|
||||
// both stay in the `full` branch only.
|
||||
safe_warn!(
|
||||
safe: (
|
||||
"Failed to start watching {} for global rules",
|
||||
source.name()
|
||||
),
|
||||
full: (
|
||||
"Failed to start watching {} for global rules: {err}",
|
||||
subdir_for_log.display()
|
||||
)
|
||||
);
|
||||
// Remove the stored watcher since registration failed.
|
||||
if let Some(state) = me.global_rules.source_watchers.remove(&cleanup_key) {
|
||||
state.repository.update(ctx, |repo, ctx| {
|
||||
repo.stop_watching(state.subscriber_id, ctx);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Handle an incremental update for the given global source.
|
||||
fn handle_global_rules_update(
|
||||
&mut self,
|
||||
source: GlobalRuleSource,
|
||||
update: RepositoryUpdate,
|
||||
ctx: &mut ModelContext<ProjectContextModel>,
|
||||
) {
|
||||
if update.is_empty() {
|
||||
return;
|
||||
}
|
||||
let Some(home_dir) = dirs::home_dir() else {
|
||||
return;
|
||||
};
|
||||
let target_file = home_dir
|
||||
.join(source.home_subdir())
|
||||
.join(source.file_pattern());
|
||||
|
||||
let was_deleted = update.deleted.iter().any(|f| f.path == target_file)
|
||||
|| update.moved.values().any(|f| f.path == target_file);
|
||||
let was_added_or_modified = update.added_or_modified().any(|f| f.path == target_file)
|
||||
|| update.moved.keys().any(|f| f.path == target_file);
|
||||
|
||||
// If the file was deleted, remove it from the cached content and emit a change event.
|
||||
if was_deleted && self.rules.remove(&target_file).is_some() {
|
||||
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
|
||||
GlobalRulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![target_file.clone()],
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
// If the file was added or modified, spawn a read to update the cached content.
|
||||
if was_added_or_modified {
|
||||
Self::spawn_global_rule_read(target_file, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/// React to creation/deletion of the registered home subdirs at runtime.
|
||||
fn handle_home_dir_event_for_global_rules(
|
||||
&mut self,
|
||||
event: &HomeDirectoryWatcherEvent,
|
||||
ctx: &mut ModelContext<ProjectContextModel>,
|
||||
) {
|
||||
let HomeDirectoryWatcherEvent::HomeFilesChanged(fs_event) = event;
|
||||
let Some(home_dir) = dirs::home_dir() else {
|
||||
log::warn!("Home directory not found; skipping global rules home dir event");
|
||||
return;
|
||||
};
|
||||
|
||||
for source in GlobalRuleSource::iter() {
|
||||
let subdir_path = home_dir.join(source.home_subdir());
|
||||
|
||||
let subdir_deleted = fs_event.deleted.contains(&subdir_path)
|
||||
|| fs_event.moved.values().any(|v| v == &subdir_path);
|
||||
if subdir_deleted {
|
||||
if let Some(state) = self.source_watchers.remove(&subdir_path) {
|
||||
state.repository.update(ctx, |repo, ctx| {
|
||||
repo.stop_watching(state.subscriber_id, ctx);
|
||||
});
|
||||
}
|
||||
let target_file = subdir_path.join(source.file_pattern());
|
||||
if self.rules.remove(&target_file).is_some() {
|
||||
ctx.emit(ProjectContextModelEvent::GlobalRulesChanged(
|
||||
GlobalRulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![target_file],
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let subdir_added =
|
||||
fs_event.added.contains(&subdir_path) || fs_event.moved.contains_key(&subdir_path);
|
||||
if subdir_added {
|
||||
let target_file = subdir_path.join(source.file_pattern());
|
||||
// Kick off the read first, then register the watcher for subsequent edits.
|
||||
Self::spawn_global_rule_read(target_file, ctx);
|
||||
self.register_global_source_watcher(source, &subdir_path, ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Subscriber for a single global rules home subdir (e.g. `~/.agents`).
|
||||
/// Tags every update with the originating [`GlobalRuleSource`] variant so the
|
||||
/// model can dispatch to the right entry without per-source channels.
|
||||
struct GlobalRulesRepositorySubscriber {
|
||||
source: GlobalRuleSource,
|
||||
update_tx: Sender<GlobalRulesUpdate>,
|
||||
}
|
||||
|
||||
impl RepositorySubscriber for GlobalRulesRepositorySubscriber {
|
||||
fn on_scan(
|
||||
&mut self,
|
||||
_repository: &Repository,
|
||||
_ctx: &mut ModelContext<Repository>,
|
||||
) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
|
||||
// Initial-state read is performed separately by `spawn_global_rule_read`,
|
||||
// so the on_scan event is intentionally a no-op.
|
||||
Box::pin(async {})
|
||||
}
|
||||
|
||||
fn on_files_updated(
|
||||
&mut self,
|
||||
_repository: &Repository,
|
||||
update: &repo_metadata::RepositoryUpdate,
|
||||
_ctx: &mut ModelContext<Repository>,
|
||||
) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
|
||||
let tx = self.update_tx.clone();
|
||||
let source = self.source;
|
||||
let update = update.clone();
|
||||
Box::pin(async move {
|
||||
let _ = tx.send(GlobalRulesUpdate { source, update }).await;
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1 +1,10 @@
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "local_fs")] {
|
||||
mod global_rules;
|
||||
pub(crate) use global_rules::GlobalRules;
|
||||
} else {
|
||||
mod dummy_global_rules;
|
||||
pub(crate) use dummy_global_rules::GlobalRules;
|
||||
}
|
||||
}
|
||||
pub mod model;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,41 @@
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use warp_util::host_id::HostId;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use warp_util::remote_path::RemotePath;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
fn local_path(path: &str) -> LocalOrRemotePath {
|
||||
LocalOrRemotePath::Local(PathBuf::from(path))
|
||||
}
|
||||
|
||||
fn insert_remote_project_rule(
|
||||
model: &mut ProjectContextModel,
|
||||
host_id: &str,
|
||||
project_root: &str,
|
||||
rule_path: &str,
|
||||
content: &str,
|
||||
) {
|
||||
let rules = model
|
||||
.path_to_rules
|
||||
.entry(remote_path(host_id, project_root))
|
||||
.or_default();
|
||||
rules.upsert_rule(&remote_path(host_id, rule_path), content.to_string());
|
||||
}
|
||||
|
||||
fn remote_path(host_id: &str, path: &str) -> LocalOrRemotePath {
|
||||
LocalOrRemotePath::Remote(RemotePath::new(
|
||||
HostId::new(host_id.to_string()),
|
||||
StandardizedPath::try_new(path).unwrap(),
|
||||
))
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_find_applicable_rules_empty_rules() {
|
||||
let rules = ProjectRules { rules: vec![] };
|
||||
let path = PathBuf::from("/a/b/c/file.rs");
|
||||
let path = local_path("/a/b/c/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert!(result.is_empty());
|
||||
@@ -14,10 +45,10 @@ fn test_find_applicable_rules_empty_rules() {
|
||||
fn test_find_applicable_rules_no_matching_rules() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/x/y/WARP.md"), "content1".to_string());
|
||||
rules.upsert_rule(Path::new("/z/AGENTS.md"), "content2".to_string());
|
||||
rules.upsert_rule(&local_path("/x/y/WARP.md"), "content1".to_string());
|
||||
rules.upsert_rule(&local_path("/z/AGENTS.md"), "content2".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/c/file.rs");
|
||||
let path = local_path("/a/b/c/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert!(result.is_empty());
|
||||
@@ -27,52 +58,52 @@ fn test_find_applicable_rules_no_matching_rules() {
|
||||
fn test_find_applicable_rules_single_matching_rule() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "content1".to_string());
|
||||
rules.upsert_rule(Path::new("/x/AGENTS.md"), "content2".to_string());
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "content1".to_string());
|
||||
rules.upsert_rule(&local_path("/x/AGENTS.md"), "content2".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/c/file.rs");
|
||||
let path = local_path("/a/b/c/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert_eq!(result[0].path, local_path("/a/WARP.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_applicable_rules_includes_all_ancestor_rules() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "root_warp".to_string());
|
||||
rules.upsert_rule(Path::new("/a/b/WARP.md"), "nested_warp".to_string());
|
||||
rules.upsert_rule(Path::new("/a/b/c/WARP.md"), "deep_warp".to_string());
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "root_warp".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/WARP.md"), "nested_warp".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/c/WARP.md"), "deep_warp".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/c/d/file.rs");
|
||||
let path = local_path("/a/b/c/d/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 3);
|
||||
|
||||
// All should be WARP.md files (same priority), order is not specified by depth
|
||||
// Just verify all expected rules are present
|
||||
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
|
||||
assert!(paths.contains(&PathBuf::from("/a/WARP.md")));
|
||||
assert!(paths.contains(&PathBuf::from("/a/b/WARP.md")));
|
||||
assert!(paths.contains(&PathBuf::from("/a/b/c/WARP.md")));
|
||||
let paths: Vec<LocalOrRemotePath> = result.iter().map(|r| r.path.clone()).collect();
|
||||
assert!(paths.contains(&local_path("/a/WARP.md")));
|
||||
assert!(paths.contains(&local_path("/a/b/WARP.md")));
|
||||
assert!(paths.contains(&local_path("/a/b/c/WARP.md")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_applicable_rules_multiple_patterns() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "agents_content".to_string());
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "warp_content".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/AGENTS.md"), "agents_content".to_string());
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "warp_content".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/file.rs");
|
||||
let path = local_path("/a/b/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 2);
|
||||
|
||||
assert_eq!(result[0].path, PathBuf::from("/a/b/AGENTS.md"));
|
||||
assert_eq!(result[0].path, local_path("/a/b/AGENTS.md"));
|
||||
assert_eq!(result[0].content, "agents_content");
|
||||
assert_eq!(result[1].path, PathBuf::from("/a/WARP.md"));
|
||||
assert_eq!(result[1].path, local_path("/a/WARP.md"));
|
||||
assert_eq!(result[1].content, "warp_content");
|
||||
}
|
||||
|
||||
@@ -80,13 +111,13 @@ fn test_find_applicable_rules_multiple_patterns() {
|
||||
fn test_find_applicable_rules_exact_path_match() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/b/WARP.md"), "exact_match".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/WARP.md"), "exact_match".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/file.rs");
|
||||
let path = local_path("/a/b/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].path, PathBuf::from("/a/b/WARP.md"));
|
||||
assert_eq!(result[0].path, local_path("/a/b/WARP.md"));
|
||||
assert_eq!(result[0].content, "exact_match");
|
||||
}
|
||||
|
||||
@@ -94,14 +125,14 @@ fn test_find_applicable_rules_exact_path_match() {
|
||||
fn test_find_applicable_rules_ignores_deeper_paths() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "applicable".to_string());
|
||||
rules.upsert_rule(Path::new("/a/b/c/d/e/WARP.md"), "too_deep".to_string()); // Path doesn't contain /a/b
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "applicable".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/c/d/e/WARP.md"), "too_deep".to_string()); // Path doesn't contain /a/b
|
||||
|
||||
let path = PathBuf::from("/a/b/file.rs");
|
||||
let path = local_path("/a/b/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert_eq!(result[0].path, local_path("/a/WARP.md"));
|
||||
assert_eq!(result[0].content, "applicable");
|
||||
}
|
||||
|
||||
@@ -109,13 +140,13 @@ fn test_find_applicable_rules_ignores_deeper_paths() {
|
||||
fn test_find_applicable_rules_handles_root_path() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/WARP.md"), "root_rule".to_string());
|
||||
rules.upsert_rule(&local_path("/WARP.md"), "root_rule".to_string());
|
||||
|
||||
let path = PathBuf::from("/a/b/file.rs");
|
||||
let path = local_path("/a/b/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].path, PathBuf::from("/WARP.md"));
|
||||
assert_eq!(result[0].path, local_path("/WARP.md"));
|
||||
assert_eq!(result[0].content, "root_rule");
|
||||
}
|
||||
|
||||
@@ -129,36 +160,36 @@ fn test_find_applicable_rules_complex_scenario() {
|
||||
// All ancestor rule files should be included.
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "a_warp".to_string());
|
||||
rules.upsert_rule(Path::new("/a/AGENTS.md"), "a_agents".to_string());
|
||||
rules.upsert_rule(Path::new("/a/b/WARP.md"), "ab_warp".to_string());
|
||||
rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "ab_agents".to_string());
|
||||
rules.upsert_rule(Path::new("/x/WARP.md"), "irrelevant".to_string()); // Should be ignored
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "a_warp".to_string());
|
||||
rules.upsert_rule(&local_path("/a/AGENTS.md"), "a_agents".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/WARP.md"), "ab_warp".to_string());
|
||||
rules.upsert_rule(&local_path("/a/b/AGENTS.md"), "ab_agents".to_string());
|
||||
rules.upsert_rule(&local_path("/x/WARP.md"), "irrelevant".to_string()); // Should be ignored
|
||||
|
||||
let path = PathBuf::from("/a/b/c/file.rs");
|
||||
let path = local_path("/a/b/c/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 4);
|
||||
|
||||
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
|
||||
assert!(paths.contains(&PathBuf::from("/a/WARP.md")));
|
||||
assert!(paths.contains(&PathBuf::from("/a/AGENTS.md")));
|
||||
assert!(paths.contains(&PathBuf::from("/a/b/WARP.md")));
|
||||
assert!(paths.contains(&PathBuf::from("/a/b/AGENTS.md")));
|
||||
// Expect only WARP.md files to be included as they have higher priority.
|
||||
assert_eq!(result[0].path, local_path("/a/WARP.md"));
|
||||
assert_eq!(result[0].content, "a_warp");
|
||||
assert_eq!(result[1].path, local_path("/a/b/WARP.md"));
|
||||
assert_eq!(result[1].content, "ab_warp");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_applicable_rules_handles_unknown_file_patterns() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("/a/WARP.md"), "known_pattern".to_string());
|
||||
rules.upsert_rule(Path::new("/a/UNKNOWN.md"), "unknown_pattern".to_string());
|
||||
let path = PathBuf::from("/a/file.rs");
|
||||
rules.upsert_rule(&local_path("/a/WARP.md"), "known_pattern".to_string());
|
||||
rules.upsert_rule(&local_path("/a/UNKNOWN.md"), "unknown_pattern".to_string());
|
||||
let path = local_path("/a/file.rs");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 1);
|
||||
|
||||
assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert_eq!(result[0].path, local_path("/a/WARP.md"));
|
||||
assert_eq!(result[0].content, "known_pattern");
|
||||
}
|
||||
|
||||
@@ -166,20 +197,459 @@ fn test_find_applicable_rules_handles_unknown_file_patterns() {
|
||||
fn test_find_applicable_rules_with_relative_paths() {
|
||||
let mut rules = ProjectRules::default();
|
||||
|
||||
rules.upsert_rule(Path::new("src/WARP.md"), "src_warp".to_string());
|
||||
rules.upsert_rule(&local_path("src/WARP.md"), "src_warp".to_string());
|
||||
rules.upsert_rule(
|
||||
Path::new("src/components/WARP.md"),
|
||||
&local_path("src/components/WARP.md"),
|
||||
"components_warp".to_string(),
|
||||
);
|
||||
|
||||
let path = PathBuf::from("src/components/Button.tsx");
|
||||
let path = local_path("src/components/Button.tsx");
|
||||
|
||||
let result = rules.find_active_or_applicable_rules(&path).active_rules;
|
||||
assert_eq!(result.len(), 2);
|
||||
|
||||
// Both are WARP.md files (same priority), order within same priority is not guaranteed
|
||||
// Just verify both rules are present
|
||||
let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
|
||||
assert!(paths.contains(&PathBuf::from("src/WARP.md")));
|
||||
assert!(paths.contains(&PathBuf::from("src/components/WARP.md")));
|
||||
let paths: Vec<LocalOrRemotePath> = result.iter().map(|r| r.path.clone()).collect();
|
||||
assert!(paths.contains(&local_path("src/WARP.md")));
|
||||
assert!(paths.contains(&local_path("src/components/WARP.md")));
|
||||
}
|
||||
|
||||
fn make_rule_path(path: &str) -> ProjectRulePath {
|
||||
ProjectRulePath {
|
||||
path: PathBuf::from(path),
|
||||
project_root: PathBuf::from("/project"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_independent_deltas() {
|
||||
let mut delta = RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
};
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/b/WARP.md")],
|
||||
});
|
||||
|
||||
assert_eq!(delta.discovered_rules.len(), 1);
|
||||
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/b/WARP.md")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_add_then_delete_yields_delete() {
|
||||
let mut delta = RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
};
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
|
||||
});
|
||||
|
||||
assert!(delta.discovered_rules.is_empty());
|
||||
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/a/WARP.md")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_delete_then_add_yields_add() {
|
||||
let mut delta = RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
|
||||
};
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
});
|
||||
|
||||
assert_eq!(delta.discovered_rules.len(), 1);
|
||||
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert!(delta.deleted_rules.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_add_delete_add_yields_add() {
|
||||
let mut delta = RulesDelta::default();
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
});
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
|
||||
});
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
});
|
||||
|
||||
assert_eq!(delta.discovered_rules.len(), 1);
|
||||
assert_eq!(delta.discovered_rules[0].path, PathBuf::from("/a/WARP.md"));
|
||||
assert!(delta.deleted_rules.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_delete_add_delete_yields_delete() {
|
||||
let mut delta = RulesDelta::default();
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
|
||||
});
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
});
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![],
|
||||
deleted_rules: vec![PathBuf::from("/a/WARP.md")],
|
||||
});
|
||||
|
||||
assert!(delta.discovered_rules.is_empty());
|
||||
assert_eq!(delta.deleted_rules, vec![PathBuf::from("/a/WARP.md")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_rediscovery_keeps_latest() {
|
||||
let mut delta = RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
};
|
||||
// A second discovery of the same path (content update) should deduplicate.
|
||||
delta.merge(RulesDelta {
|
||||
discovered_rules: vec![make_rule_path("/a/WARP.md")],
|
||||
deleted_rules: vec![],
|
||||
});
|
||||
|
||||
assert_eq!(delta.discovered_rules.len(), 1);
|
||||
assert!(delta.deleted_rules.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_missing_rule_content_preserves_cached_content_while_path_is_standing() {
|
||||
let rule_path = local_path("/unavailable/project/WARP.md");
|
||||
let mut existing_rules = ProjectRules::default();
|
||||
existing_rules.upsert_rule(&rule_path, "cached content".to_string());
|
||||
|
||||
let rules = ProjectContextModel::reconcile_project_rules(
|
||||
vec![rule_path.clone()],
|
||||
Vec::new(),
|
||||
existing_rules,
|
||||
);
|
||||
let result = rules.find_active_or_applicable_rules(&local_path("/unavailable/project/main.rs"));
|
||||
|
||||
assert_eq!(result.active_rules.len(), 1);
|
||||
assert_eq!(result.active_rules[0].path, rule_path);
|
||||
assert_eq!(result.active_rules[0].content, "cached content");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rule_missing_from_standing_results_is_removed_from_cached_content() {
|
||||
let rule_path = local_path("/unavailable/project/WARP.md");
|
||||
let mut existing_rules = ProjectRules::default();
|
||||
existing_rules.upsert_rule(&rule_path, "cached content".to_string());
|
||||
|
||||
let rules =
|
||||
ProjectContextModel::reconcile_project_rules(Vec::new(), Vec::new(), existing_rules);
|
||||
assert!(rules.rule_paths().next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconcile_project_rules_hydrates_local_and_remote_paths() {
|
||||
let local_rule_path = local_path("/local/WARP.md");
|
||||
let remote_rule_path = remote_path("host-a", "/remote/AGENTS.md");
|
||||
|
||||
let rules = ProjectContextModel::reconcile_project_rules(
|
||||
vec![local_rule_path.clone(), remote_rule_path.clone()],
|
||||
vec![
|
||||
(local_rule_path.clone(), "local content".to_string()),
|
||||
(remote_rule_path.clone(), "remote content".to_string()),
|
||||
],
|
||||
ProjectRules::default(),
|
||||
);
|
||||
|
||||
let local_result = rules.find_active_or_applicable_rules(&local_path("/local/main.rs"));
|
||||
assert_eq!(local_result.active_rules.len(), 1);
|
||||
assert_eq!(local_result.active_rules[0].path, local_rule_path);
|
||||
assert_eq!(local_result.active_rules[0].content, "local content");
|
||||
|
||||
let remote_result =
|
||||
rules.find_active_or_applicable_rules(&remote_path("host-a", "/remote/main.rs"));
|
||||
assert_eq!(remote_result.active_rules.len(), 1);
|
||||
assert_eq!(remote_result.active_rules[0].path, remote_rule_path);
|
||||
assert_eq!(remote_result.active_rules[0].content, "remote content");
|
||||
}
|
||||
|
||||
#[cfg(feature = "local_fs")]
|
||||
#[test]
|
||||
fn test_remote_standing_results_preserve_host_qualified_rule_paths() {
|
||||
let host = HostId::new("test-host".to_string());
|
||||
let repo_id = RepositoryIdentifier::Remote(RemotePath::new(
|
||||
host.clone(),
|
||||
StandardizedPath::try_new("/repo").unwrap(),
|
||||
));
|
||||
let rule_path = StandardizedPath::try_new("/repo/nested/WARP.md").unwrap();
|
||||
let contents = [
|
||||
StandingQueryContent::file(rule_path.clone()),
|
||||
StandingQueryContent::directory(StandardizedPath::try_new("/repo/nested").unwrap()),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
standing_project_rule_paths(&repo_id, &contents),
|
||||
vec![LocalOrRemotePath::Remote(RemotePath::new(host, rule_path))]
|
||||
);
|
||||
}
|
||||
|
||||
// Helper for global-rules tests: inserts a synthetic global rule directly into
|
||||
// the model. Bypasses the watcher infrastructure (which requires the warpui
|
||||
// runtime) so we can exercise `find_applicable_rules`'s layering logic.
|
||||
fn insert_global_rule(model: &mut ProjectContextModel, path: &Path, content: &str) {
|
||||
model.global_rules.rules.insert(
|
||||
path.to_path_buf(),
|
||||
ProjectRule {
|
||||
path: LocalOrRemotePath::Local(path.to_path_buf()),
|
||||
content: content.to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_project_rule(
|
||||
model: &mut ProjectContextModel,
|
||||
project_root: &Path,
|
||||
rule_path: &Path,
|
||||
content: &str,
|
||||
) {
|
||||
let rules = model
|
||||
.path_to_rules
|
||||
.entry(LocalOrRemotePath::Local(project_root.to_path_buf()))
|
||||
.or_default();
|
||||
rules.upsert_rule(
|
||||
&LocalOrRemotePath::Local(rule_path.to_path_buf()),
|
||||
content.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remote_project_rules_require_matching_host() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_remote_project_rule(
|
||||
&mut model,
|
||||
"host-a",
|
||||
"/repo",
|
||||
"/repo/WARP.md",
|
||||
"remote_project_rule",
|
||||
);
|
||||
|
||||
let same_host = model
|
||||
.find_applicable_project_rules(&remote_path("host-a", "/repo/src/main.rs"))
|
||||
.expect("same-host remote rule should apply");
|
||||
assert_eq!(same_host.root_path, remote_path("host-a", "/repo"));
|
||||
assert_eq!(same_host.active_rules.len(), 1);
|
||||
assert_eq!(same_host.active_rules[0].content, "remote_project_rule");
|
||||
|
||||
let other_host =
|
||||
model.find_applicable_project_rules(&remote_path("host-b", "/repo/src/main.rs"));
|
||||
assert!(other_host.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_rule_alone_no_project_rules() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(
|
||||
&mut model,
|
||||
Path::new("/home/u/.agents/AGENTS.md"),
|
||||
"global_content",
|
||||
);
|
||||
|
||||
let result = model
|
||||
.find_applicable_rules(&local_path("/some/project/file.rs"))
|
||||
.expect("global rule should produce a result");
|
||||
|
||||
assert_eq!(result.active_rules.len(), 1);
|
||||
assert_eq!(
|
||||
result.active_rules[0].path,
|
||||
local_path("/home/u/.agents/AGENTS.md")
|
||||
);
|
||||
assert_eq!(result.active_rules[0].content, "global_content");
|
||||
assert!(result.additional_rule_paths.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_rule_layered_with_project_warp() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
|
||||
insert_project_rule(
|
||||
&mut model,
|
||||
Path::new("/repo"),
|
||||
Path::new("/repo/WARP.md"),
|
||||
"project_warp",
|
||||
);
|
||||
|
||||
let result = model
|
||||
.find_applicable_rules(&local_path("/repo/src/main.rs"))
|
||||
.expect("layered rules should produce a result");
|
||||
|
||||
// Layered precedence: global first, then project rules.
|
||||
assert_eq!(result.active_rules.len(), 2);
|
||||
assert_eq!(result.active_rules[0].content, "global");
|
||||
assert_eq!(result.active_rules[1].content, "project_warp");
|
||||
assert_eq!(result.root_path, local_path("/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_in_dir_warp_shadows_agents_with_global() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
|
||||
// Both WARP.md and AGENTS.md in the same project directory: WARP.md should
|
||||
// shadow AGENTS.md (existing in-directory behavior preserved).
|
||||
insert_project_rule(
|
||||
&mut model,
|
||||
Path::new("/repo"),
|
||||
Path::new("/repo/WARP.md"),
|
||||
"project_warp",
|
||||
);
|
||||
insert_project_rule(
|
||||
&mut model,
|
||||
Path::new("/repo"),
|
||||
Path::new("/repo/AGENTS.md"),
|
||||
"project_agents",
|
||||
);
|
||||
|
||||
let result = model
|
||||
.find_applicable_rules(&local_path("/repo/src/main.rs"))
|
||||
.expect("layered rules should produce a result");
|
||||
|
||||
// Expect: [global, project WARP.md]. project AGENTS.md is shadowed.
|
||||
assert_eq!(result.active_rules.len(), 2);
|
||||
assert_eq!(result.active_rules[0].content, "global");
|
||||
assert_eq!(result.active_rules[1].content, "project_warp");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_rules_returns_none() {
|
||||
let model = ProjectContextModel::default();
|
||||
let result = model.find_applicable_rules(&local_path("/some/path/file.rs"));
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_rule_root_path_falls_back_to_parent() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(&mut model, Path::new("/home/u/.agents/AGENTS.md"), "global");
|
||||
|
||||
let result = model
|
||||
.find_applicable_rules(&local_path("/some/file.rs"))
|
||||
.expect("global rule should produce a result");
|
||||
|
||||
// No project root indexed; root_path falls back to parent of the global rule.
|
||||
assert_eq!(result.root_path, local_path("/home/u/.agents"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_global_rules_all_contribute() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(
|
||||
&mut model,
|
||||
Path::new("/home/u/.agents/AGENTS.md"),
|
||||
"agents_global",
|
||||
);
|
||||
insert_global_rule(
|
||||
&mut model,
|
||||
Path::new("/home/u/.warp/WARP.md"),
|
||||
"warp_global",
|
||||
);
|
||||
|
||||
let result = model
|
||||
.find_applicable_rules(&local_path("/repo/src/main.rs"))
|
||||
.expect("globals should produce a result");
|
||||
|
||||
assert_eq!(result.active_rules.len(), 2);
|
||||
let contents: Vec<&str> = result
|
||||
.active_rules
|
||||
.iter()
|
||||
.map(|r| r.content.as_str())
|
||||
.collect();
|
||||
assert!(contents.contains(&"agents_global"));
|
||||
assert!(contents.contains(&"warp_global"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remote_global_rules_only_layer_for_matching_remote_host() {
|
||||
let mut model = ProjectContextModel::default();
|
||||
insert_global_rule(
|
||||
&mut model,
|
||||
Path::new("/home/local/.agents/AGENTS.md"),
|
||||
"local_global",
|
||||
);
|
||||
insert_remote_project_rule(
|
||||
&mut model,
|
||||
"host-a",
|
||||
"/repo",
|
||||
"/repo/WARP.md",
|
||||
"remote_project",
|
||||
);
|
||||
let host_a = HostId::new("host-a".to_string());
|
||||
model.set_remote_global_rules(
|
||||
host_a.clone(),
|
||||
vec![ProjectRule {
|
||||
path: remote_path("host-a", "/home/remote/.agents/AGENTS.md"),
|
||||
content: "remote_global".to_string(),
|
||||
}],
|
||||
);
|
||||
model.set_remote_global_rules(
|
||||
HostId::new("host-b".to_string()),
|
||||
vec![ProjectRule {
|
||||
path: remote_path("host-b", "/home/remote/.agents/AGENTS.md"),
|
||||
content: "other_remote_global".to_string(),
|
||||
}],
|
||||
);
|
||||
|
||||
let matching = model
|
||||
.find_applicable_rules(&remote_path("host-a", "/repo/src/main.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
matching
|
||||
.active_rules
|
||||
.iter()
|
||||
.map(|rule| rule.content.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["local_global", "remote_global", "remote_project"]
|
||||
);
|
||||
|
||||
let other_host = model
|
||||
.find_applicable_rules(&remote_path("host-b", "/repo/src/main.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
other_host
|
||||
.active_rules
|
||||
.iter()
|
||||
.map(|rule| rule.content.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["local_global", "other_remote_global"]
|
||||
);
|
||||
|
||||
let local = model
|
||||
.find_applicable_rules(&local_path("/repo/src/main.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(local.active_rules.len(), 1);
|
||||
assert_eq!(local.active_rules[0].content, "local_global");
|
||||
|
||||
assert_eq!(
|
||||
model.global_rule_paths().collect::<Vec<_>>(),
|
||||
[local_path("/home/local/.agents/AGENTS.md")]
|
||||
);
|
||||
|
||||
model.set_remote_global_rules(host_a, Vec::new());
|
||||
let replaced = model
|
||||
.find_applicable_rules(&remote_path("host-a", "/repo/src/main.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replaced
|
||||
.active_rules
|
||||
.iter()
|
||||
.map(|rule| rule.content.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["local_global", "remote_project"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::{
|
||||
agent::action_result::{AnyFileContent, FileContext},
|
||||
skills::{ParsedSkill, SkillProvider, SkillScope},
|
||||
};
|
||||
use warp_multi_agent_api as api;
|
||||
|
||||
use thiserror::Error;
|
||||
use warp_multi_agent_api as api;
|
||||
use warp_util::host_id::HostId;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use warp_util::remote_path::RemotePath;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
use crate::agent::action_result::{AnyFileContent, FileContext};
|
||||
use crate::skills::{ParsedSkill, SkillProvider, SkillReference, SkillScope};
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum SkillConversionError {
|
||||
@@ -22,14 +24,126 @@ pub enum SkillConversionError {
|
||||
ProviderInvalid,
|
||||
#[error("Invalid content")]
|
||||
ContentInvalid,
|
||||
#[error("Skill path origin is unavailable")]
|
||||
PathOriginUnavailable,
|
||||
#[error("Invalid remote skill path")]
|
||||
RemotePathInvalid,
|
||||
}
|
||||
/// Identifies how a string skill path from an API payload should be interpreted.
|
||||
///
|
||||
/// Live agent responses can be decoded from the active session's location. Restored payloads do
|
||||
/// not carry enough session identity to safely reconstruct path-based skill locations, so callers
|
||||
/// must use [`SkillPathOrigin::Unavailable`] rather than silently assuming the local filesystem.
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub enum SkillPathOrigin {
|
||||
Local,
|
||||
Remote {
|
||||
host_id: HostId,
|
||||
},
|
||||
/// Path identity could not be restored, but the API payload already carries the skill
|
||||
/// descriptor and content needed to render a historical transcript.
|
||||
///
|
||||
/// This intentionally uses a local path wrapper only as a display-compatible identity for
|
||||
/// restored conversation UI. Live execution paths should use [`SkillPathOrigin::Local`] or
|
||||
/// [`SkillPathOrigin::Remote`] so local/remote provenance is preserved.
|
||||
RestoredDisplayOnly,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
impl SkillPathOrigin {
|
||||
pub fn location_for_path(
|
||||
&self,
|
||||
path: impl Into<String>,
|
||||
) -> Result<LocalOrRemotePath, SkillConversionError> {
|
||||
let path = path.into();
|
||||
match self {
|
||||
SkillPathOrigin::Local | SkillPathOrigin::RestoredDisplayOnly => {
|
||||
// Normalize the path to collapse duplicate separators (e.g. `//workspace/...`
|
||||
// → `/workspace/...`) so skill cache lookups match the filesystem-derived keys.
|
||||
// We operate on the raw string rather than using `PathBuf::components().collect()`
|
||||
// because the latter re-serialises with platform-specific separators (backslashes
|
||||
// on Windows) and treats leading `//` as a UNC prefix on Windows.
|
||||
let normalized = collapse_slashes(&path);
|
||||
Ok(LocalOrRemotePath::Local(PathBuf::from(normalized)))
|
||||
}
|
||||
SkillPathOrigin::Remote { host_id } => {
|
||||
let path = StandardizedPath::try_new(&path)
|
||||
.map_err(|_| SkillConversionError::RemotePathInvalid)?;
|
||||
Ok(LocalOrRemotePath::Remote(RemotePath::new(
|
||||
host_id.clone(),
|
||||
path,
|
||||
)))
|
||||
}
|
||||
SkillPathOrigin::Unavailable => Err(SkillConversionError::PathOriginUnavailable),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collapse consecutive `/` separators into a single one.
|
||||
///
|
||||
/// Skill paths are always forward-slash POSIX-style paths on all platforms, so we normalise
|
||||
/// at the string level rather than using [`std::path::PathBuf::components`], which would
|
||||
/// re-serialise with backslashes on Windows and misinterpret `//prefix` as a UNC path.
|
||||
fn collapse_slashes(path: &str) -> String {
|
||||
let mut result = String::with_capacity(path.len());
|
||||
let mut prev_slash = false;
|
||||
for ch in path.chars() {
|
||||
if ch == '/' {
|
||||
if !prev_slash {
|
||||
result.push(ch);
|
||||
}
|
||||
prev_slash = true;
|
||||
} else {
|
||||
result.push(ch);
|
||||
prev_slash = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn skill_reference_for_path(
|
||||
path: impl Into<String>,
|
||||
path_origin: &SkillPathOrigin,
|
||||
) -> Result<SkillReference, SkillConversionError> {
|
||||
path_origin
|
||||
.location_for_path(path)
|
||||
.map(SkillReference::Path)
|
||||
}
|
||||
|
||||
pub fn skill_reference_from_api_skill_ref(
|
||||
skill_ref: api::SkillRef,
|
||||
path_origin: &SkillPathOrigin,
|
||||
) -> Option<SkillReference> {
|
||||
match skill_ref.skill_reference {
|
||||
Some(api::skill_ref::SkillReference::Path(path)) => {
|
||||
skill_reference_for_path(path, path_origin).ok()
|
||||
}
|
||||
Some(api::skill_ref::SkillReference::BundledSkillId(id)) => {
|
||||
Some(SkillReference::BundledSkillId(id))
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn skill_reference_from_read_skill_ref(
|
||||
skill_reference: api::message::tool_call::read_skill::SkillReference,
|
||||
path_origin: &SkillPathOrigin,
|
||||
) -> Result<SkillReference, SkillConversionError> {
|
||||
match skill_reference {
|
||||
api::message::tool_call::read_skill::SkillReference::SkillPath(path) => {
|
||||
skill_reference_for_path(path, path_origin)
|
||||
}
|
||||
api::message::tool_call::read_skill::SkillReference::BundledSkillId(id) => {
|
||||
Ok(SkillReference::BundledSkillId(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
impl From<ParsedSkill> for api::Skill {
|
||||
fn from(skill: ParsedSkill) -> Self {
|
||||
api::Skill {
|
||||
descriptor: Some(api::SkillDescriptor {
|
||||
skill_reference: Some(api::skill_descriptor::SkillReference::Path(
|
||||
skill.path.to_string_lossy().to_string(),
|
||||
skill.path.display_path(),
|
||||
)),
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
@@ -37,7 +151,7 @@ impl From<ParsedSkill> for api::Skill {
|
||||
provider: Some(skill.provider.into()),
|
||||
}),
|
||||
content: Some(api::FileContent {
|
||||
file_path: skill.path.to_string_lossy().to_string(),
|
||||
file_path: skill.path.display_path(),
|
||||
content: skill.content,
|
||||
line_range: skill
|
||||
.line_range
|
||||
@@ -89,6 +203,15 @@ impl TryFrom<api::Skill> for ParsedSkill {
|
||||
type Error = SkillConversionError;
|
||||
|
||||
fn try_from(api_skill: api::Skill) -> Result<Self, Self::Error> {
|
||||
Self::try_from_api_with_origin(api_skill, &SkillPathOrigin::Unavailable)
|
||||
}
|
||||
}
|
||||
|
||||
impl ParsedSkill {
|
||||
pub fn try_from_api_with_origin(
|
||||
api_skill: api::Skill,
|
||||
path_origin: &SkillPathOrigin,
|
||||
) -> Result<Self, SkillConversionError> {
|
||||
let Some(descriptor) = api_skill.descriptor else {
|
||||
return Err(SkillConversionError::MissingDescriptor);
|
||||
};
|
||||
@@ -121,7 +244,7 @@ impl TryFrom<api::Skill> for ParsedSkill {
|
||||
let line_range = context.line_range.as_ref();
|
||||
|
||||
Ok(ParsedSkill {
|
||||
path: PathBuf::from(&path),
|
||||
path: path_origin.location_for_path(path)?,
|
||||
name: descriptor.name,
|
||||
description: descriptor.description,
|
||||
content,
|
||||
@@ -164,3 +287,7 @@ fn convert_provider(
|
||||
api::skill_descriptor::provider::Type::OpenCode(_) => Ok(SkillProvider::OpenCode),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "conversion_tests.rs"]
|
||||
mod conversion_tests;
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
use warp_multi_agent_api as api;
|
||||
use warp_util::host_id::HostId;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use warp_util::remote_path::RemotePath;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
use super::{
|
||||
skill_reference_from_api_skill_ref, skill_reference_from_read_skill_ref, SkillConversionError,
|
||||
SkillPathOrigin,
|
||||
};
|
||||
use crate::skills::{ParsedSkill, SkillProvider, SkillReference, SkillScope};
|
||||
|
||||
fn api_project_skill(path: &str) -> api::Skill {
|
||||
api::Skill {
|
||||
descriptor: Some(api::SkillDescriptor {
|
||||
skill_reference: Some(api::skill_descriptor::SkillReference::Path(
|
||||
path.to_string(),
|
||||
)),
|
||||
name: "deploy".to_string(),
|
||||
description: "Deploy the service".to_string(),
|
||||
scope: Some(api::skill_descriptor::Scope {
|
||||
r#type: Some(api::skill_descriptor::scope::Type::Project(())),
|
||||
}),
|
||||
provider: Some(api::skill_descriptor::Provider {
|
||||
r#type: Some(api::skill_descriptor::provider::Type::Agents(())),
|
||||
}),
|
||||
}),
|
||||
content: Some(api::FileContent {
|
||||
file_path: path.to_string(),
|
||||
content: "# Deploy".to_string(),
|
||||
line_range: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_from_api_with_remote_origin_preserves_host_identity() {
|
||||
let host_id = HostId::new("remote-host".to_string());
|
||||
let parsed = ParsedSkill::try_from_api_with_origin(
|
||||
api_project_skill("/repo/.agents/skills/deploy/SKILL.md"),
|
||||
&SkillPathOrigin::Remote {
|
||||
host_id: host_id.clone(),
|
||||
},
|
||||
)
|
||||
.expect("remote project skill should convert");
|
||||
|
||||
let LocalOrRemotePath::Remote(path) = parsed.path else {
|
||||
panic!("expected a remote skill path");
|
||||
};
|
||||
assert_eq!(path.host_id, host_id);
|
||||
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skill_ref_with_remote_origin_preserves_host_identity() {
|
||||
let host_id = HostId::new("remote-host".to_string());
|
||||
let skill_reference = skill_reference_from_api_skill_ref(
|
||||
api::SkillRef {
|
||||
skill_reference: Some(api::skill_ref::SkillReference::Path(
|
||||
"/repo/.agents/skills/deploy/SKILL.md".to_string(),
|
||||
)),
|
||||
},
|
||||
&SkillPathOrigin::Remote {
|
||||
host_id: host_id.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let Some(SkillReference::Path(LocalOrRemotePath::Remote(path))) = skill_reference else {
|
||||
panic!("expected a remote skill path");
|
||||
};
|
||||
assert_eq!(path.host_id, host_id);
|
||||
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsed_skill_api_conversion_emits_plain_path_reference() {
|
||||
let skill_path = LocalOrRemotePath::Remote(RemotePath::new(
|
||||
HostId::new("remote-host".to_string()),
|
||||
StandardizedPath::try_new("/repo/.agents/skills/deploy/SKILL.md").unwrap(),
|
||||
));
|
||||
let api_skill: api::Skill = ParsedSkill {
|
||||
path: skill_path.clone(),
|
||||
name: "deploy".to_string(),
|
||||
description: "Deploy the service".to_string(),
|
||||
content: "# Deploy".to_string(),
|
||||
line_range: None,
|
||||
scope: SkillScope::Project,
|
||||
provider: SkillProvider::Agents,
|
||||
}
|
||||
.into();
|
||||
|
||||
let descriptor = api_skill
|
||||
.descriptor
|
||||
.expect("converted skill should have descriptor");
|
||||
assert_eq!(
|
||||
descriptor.skill_reference,
|
||||
Some(api::skill_descriptor::SkillReference::Path(
|
||||
skill_path.display_path()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skill_reference_api_conversion_emits_plain_path_reference() {
|
||||
let skill_path = LocalOrRemotePath::Remote(RemotePath::new(
|
||||
HostId::new("remote-host".to_string()),
|
||||
StandardizedPath::try_new("/repo/.agents/skills/deploy/SKILL.md").unwrap(),
|
||||
));
|
||||
let reference: api::skill_descriptor::SkillReference =
|
||||
SkillReference::Path(skill_path.clone()).into();
|
||||
|
||||
assert_eq!(
|
||||
reference,
|
||||
api::skill_descriptor::SkillReference::Path(skill_path.display_path())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_from_api_with_unavailable_origin_rejects_path_based_skills() {
|
||||
let error = ParsedSkill::try_from_api_with_origin(
|
||||
api_project_skill("/repo/.agents/skills/deploy/SKILL.md"),
|
||||
&SkillPathOrigin::Unavailable,
|
||||
)
|
||||
.expect_err("restored skills without host context should not fabricate local paths");
|
||||
|
||||
assert!(matches!(error, SkillConversionError::PathOriginUnavailable));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skill_ref_with_unavailable_origin_preserves_bundled_skills() {
|
||||
let skill_reference = skill_reference_from_api_skill_ref(
|
||||
api::SkillRef {
|
||||
skill_reference: Some(api::skill_ref::SkillReference::BundledSkillId(
|
||||
"review-comments".to_string(),
|
||||
)),
|
||||
},
|
||||
&SkillPathOrigin::Unavailable,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
skill_reference,
|
||||
Some(SkillReference::BundledSkillId(
|
||||
"review-comments".to_string()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_origin_normalizes_double_leading_slash() {
|
||||
let result = SkillPathOrigin::Local
|
||||
.location_for_path("//workspace/common-skills/.agents/skills/deploy/SKILL.md")
|
||||
.expect("double-slash path should be accepted");
|
||||
|
||||
let LocalOrRemotePath::Local(path) = result else {
|
||||
panic!("expected a local path");
|
||||
};
|
||||
assert_eq!(
|
||||
path.to_str().unwrap(),
|
||||
"/workspace/common-skills/.agents/skills/deploy/SKILL.md"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_origin_normalizes_multiple_slashes() {
|
||||
let result = SkillPathOrigin::Local
|
||||
.location_for_path("///workspace///skills///SKILL.md")
|
||||
.expect("multi-slash path should be accepted");
|
||||
|
||||
let LocalOrRemotePath::Local(path) = result else {
|
||||
panic!("expected a local path");
|
||||
};
|
||||
assert_eq!(path.to_str().unwrap(), "/workspace/skills/SKILL.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_origin_preserves_normal_absolute_path() {
|
||||
let result = SkillPathOrigin::Local
|
||||
.location_for_path("/workspace/.agents/skills/deploy/SKILL.md")
|
||||
.expect("normal path should be accepted");
|
||||
|
||||
let LocalOrRemotePath::Local(path) = result else {
|
||||
panic!("expected a local path");
|
||||
};
|
||||
assert_eq!(
|
||||
path.to_str().unwrap(),
|
||||
"/workspace/.agents/skills/deploy/SKILL.md"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_display_origin_normalizes_double_leading_slash() {
|
||||
let result = SkillPathOrigin::RestoredDisplayOnly
|
||||
.location_for_path("//repo/.agents/skills/deploy/SKILL.md")
|
||||
.expect("double-slash path should be accepted");
|
||||
|
||||
let LocalOrRemotePath::Local(path) = result else {
|
||||
panic!("expected a local path");
|
||||
};
|
||||
assert_eq!(
|
||||
path.to_str().unwrap(),
|
||||
"/repo/.agents/skills/deploy/SKILL.md"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_skill_ref_with_local_origin_normalizes_double_slash() {
|
||||
let skill_reference = skill_reference_from_read_skill_ref(
|
||||
api::message::tool_call::read_skill::SkillReference::SkillPath(
|
||||
"//workspace/.agents/skills/deploy/SKILL.md".to_string(),
|
||||
),
|
||||
&SkillPathOrigin::Local,
|
||||
)
|
||||
.expect("double-slash read_skill path should convert");
|
||||
|
||||
let SkillReference::Path(LocalOrRemotePath::Local(path)) = skill_reference else {
|
||||
panic!("expected a local skill path");
|
||||
};
|
||||
assert_eq!(
|
||||
path.to_str().unwrap(),
|
||||
"/workspace/.agents/skills/deploy/SKILL.md"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_skill_ref_with_remote_origin_preserves_host_identity() {
|
||||
let host_id = HostId::new("remote-host".to_string());
|
||||
let skill_reference = skill_reference_from_read_skill_ref(
|
||||
api::message::tool_call::read_skill::SkillReference::SkillPath(
|
||||
"/repo/.agents/skills/deploy/SKILL.md".to_string(),
|
||||
),
|
||||
&SkillPathOrigin::Remote {
|
||||
host_id: host_id.clone(),
|
||||
},
|
||||
)
|
||||
.expect("remote read_skill skill references should convert");
|
||||
|
||||
let SkillReference::Path(LocalOrRemotePath::Remote(path)) = skill_reference else {
|
||||
panic!("expected a remote skill path");
|
||||
};
|
||||
assert_eq!(path.host_id, host_id);
|
||||
assert_eq!(path.path.as_str(), "/repo/.agents/skills/deploy/SKILL.md");
|
||||
}
|
||||
@@ -4,11 +4,16 @@ mod parser;
|
||||
mod read_skills;
|
||||
mod skill_provider;
|
||||
mod skill_reference;
|
||||
|
||||
pub use parse_skill::{parse_bundled_skill, parse_skill, ParsedSkill};
|
||||
pub use conversion::{
|
||||
skill_reference_from_api_skill_ref, skill_reference_from_read_skill_ref, SkillConversionError,
|
||||
SkillPathOrigin,
|
||||
};
|
||||
pub use parse_skill::{
|
||||
parse_bundled_skill, parse_skill, parse_skill_content_at_location, ParsedSkill,
|
||||
};
|
||||
pub use read_skills::read_skills;
|
||||
pub use skill_provider::{
|
||||
get_provider_for_path, home_skills_path, provider_rank, SkillProvider, SkillProviderDefinition,
|
||||
SkillScope, SKILL_PROVIDER_DEFINITIONS,
|
||||
get_provider_for_path, home_skills_path, provider_parent_directory_for_skills_root,
|
||||
provider_rank, SkillProvider, SkillProviderDefinition, SkillScope, SKILL_PROVIDER_DEFINITIONS,
|
||||
};
|
||||
pub use skill_reference::SkillReference;
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
use std::fmt::Display;
|
||||
use std::fs;
|
||||
use std::ops::Range;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use std::fmt::Display;
|
||||
use std::ops::Range;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::parser::parse_markdown_file;
|
||||
use super::skill_provider::{get_provider_for_path, get_scope_for_path, SkillProvider, SkillScope};
|
||||
use thiserror::Error;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
|
||||
use super::parser::parse_markdown_content;
|
||||
use super::skill_provider::{get_provider_for_path, get_scope_for_path, SkillProvider, SkillScope};
|
||||
|
||||
const MAX_SKILL_DESCRIPTION_CHARS: usize = 512;
|
||||
|
||||
@@ -17,6 +20,50 @@ lazy_static! {
|
||||
static ref INCOMPLETE_SENTENCE: Regex =
|
||||
Regex::new(r"[^.!?]*$").expect("Incomplete sentence regex should be valid");
|
||||
}
|
||||
/// Parse skill markdown content that was fetched outside the local filesystem.
|
||||
///
|
||||
/// This is used for remote project skills, whose SKILL.md body arrives through
|
||||
/// the remote file-read transport rather than `std::fs`.
|
||||
pub fn parse_skill_content_at_location(
|
||||
path: LocalOrRemotePath,
|
||||
content: &str,
|
||||
provider: SkillProvider,
|
||||
scope: SkillScope,
|
||||
) -> Result<ParsedSkill> {
|
||||
let parsed = parse_markdown_content(content)?;
|
||||
let name = match parsed
|
||||
.front_matter
|
||||
.get("name")
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
Some(name) => name.to_string(),
|
||||
None => derive_skill_name_from_path(&path)?,
|
||||
};
|
||||
|
||||
let description = match parsed
|
||||
.front_matter
|
||||
.get("description")
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
Some(description) => description.to_string(),
|
||||
None => truncate_skill_description(
|
||||
&derive_description_from_content(&parsed.content, parsed.line_range.as_ref())
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
};
|
||||
|
||||
Ok(ParsedSkill {
|
||||
path,
|
||||
name,
|
||||
description,
|
||||
content: parsed.content,
|
||||
line_range: parsed.line_range,
|
||||
provider,
|
||||
scope,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ParseSkillError {
|
||||
@@ -29,7 +76,7 @@ pub enum ParseSkillError {
|
||||
/// Represents a parsed skill with validated fields
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ParsedSkill {
|
||||
pub path: PathBuf,
|
||||
pub path: LocalOrRemotePath,
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
/// The entire content of the file (including front matter)
|
||||
@@ -52,7 +99,7 @@ impl ParsedSkill {
|
||||
|
||||
impl Display for ParsedSkill {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Skill: {}", self.path.display())
|
||||
write!(f, "Skill: {}", self.path.display_path())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,9 +111,10 @@ impl Display for ParsedSkill {
|
||||
/// # Returns
|
||||
/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
|
||||
pub fn parse_skill(path: &Path) -> Result<ParsedSkill> {
|
||||
let provider = get_provider_for_path(path).unwrap_or(SkillProvider::Agents);
|
||||
let provider_path = LocalOrRemotePath::Local(path.to_path_buf());
|
||||
let provider = get_provider_for_path(&provider_path).unwrap_or(SkillProvider::Agents);
|
||||
let scope = get_scope_for_path(path);
|
||||
parse_skill_internal(path, provider, scope)
|
||||
parse_local_skill_internal(path, provider, scope)
|
||||
}
|
||||
|
||||
/// Parse a bundled skill markdown file.
|
||||
@@ -81,55 +129,26 @@ pub fn parse_skill(path: &Path) -> Result<ParsedSkill> {
|
||||
/// # Returns
|
||||
/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
|
||||
pub fn parse_bundled_skill(path: &Path) -> Result<ParsedSkill> {
|
||||
parse_skill_internal(path, SkillProvider::Warp, SkillScope::Bundled)
|
||||
parse_local_skill_internal(path, SkillProvider::Warp, SkillScope::Bundled)
|
||||
}
|
||||
|
||||
fn parse_skill_internal(
|
||||
fn parse_local_skill_internal(
|
||||
path: &Path,
|
||||
provider: SkillProvider,
|
||||
scope: SkillScope,
|
||||
) -> Result<ParsedSkill> {
|
||||
let parsed = parse_markdown_file(path)?;
|
||||
|
||||
let name = match parsed
|
||||
.front_matter
|
||||
.get("name")
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
Some(name) => name.to_string(),
|
||||
None => derive_skill_name_from_path(path)?,
|
||||
};
|
||||
|
||||
let description = match parsed
|
||||
.front_matter
|
||||
.get("description")
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
Some(description) => description.to_string(),
|
||||
None => truncate_skill_description(
|
||||
&derive_description_from_content(&parsed.content, parsed.line_range.as_ref())
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
};
|
||||
|
||||
Ok(ParsedSkill {
|
||||
path: path.to_path_buf(),
|
||||
name,
|
||||
description,
|
||||
content: parsed.content,
|
||||
line_range: parsed.line_range,
|
||||
let content = fs::read_to_string(path)?;
|
||||
parse_skill_content_at_location(
|
||||
LocalOrRemotePath::Local(path.to_path_buf()),
|
||||
&content,
|
||||
provider,
|
||||
scope,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn derive_skill_name_from_path(path: &Path) -> Result<String> {
|
||||
fn derive_skill_name_from_path(path: &LocalOrRemotePath) -> Result<String> {
|
||||
path.parent()
|
||||
.and_then(|parent| parent.file_name())
|
||||
.and_then(|name| name.to_str())
|
||||
.map(|name| name.to_string())
|
||||
.and_then(|parent| parent.file_name().map(str::to_owned))
|
||||
.ok_or(ParseSkillError::CouldNotDeriveSkillNameFromPath.into())
|
||||
}
|
||||
|
||||
@@ -202,5 +221,5 @@ fn truncate_skill_description(description: &str) -> String {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "parse_skill_test.rs"]
|
||||
#[path = "parse_skill_tests.rs"]
|
||||
mod parse_skill_test;
|
||||
|
||||
+3
-1
@@ -1,5 +1,7 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -48,7 +50,7 @@ Show concrete examples of using this Skill.
|
||||
// Total of 12 lines, so line_range is 5..13
|
||||
assert_eq!(result.line_range, Some(5..13));
|
||||
// Verify path is the full file path
|
||||
assert_eq!(result.path, skill_file);
|
||||
assert_eq!(result.path, LocalOrRemotePath::Local(skill_file));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1,10 +1,9 @@
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Range;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use regex::Regex;
|
||||
use serde_yaml::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::ops::Range;
|
||||
use std::path::Path;
|
||||
|
||||
/// Represents a parsed markdown file with YAML front matter
|
||||
#[derive(Debug)]
|
||||
@@ -20,19 +19,6 @@ pub struct ParsedMarkdown {
|
||||
pub line_range: Option<Range<usize>>,
|
||||
}
|
||||
|
||||
/// Parse a markdown file with YAML front matter
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `path` - Path to the markdown file to parse
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Result<ParsedMarkdown>` - Parsed document with front matter and content
|
||||
#[allow(dead_code)]
|
||||
pub fn parse_markdown_file(path: &Path) -> Result<ParsedMarkdown> {
|
||||
let content = fs::read_to_string(path)?;
|
||||
parse_markdown_content(&content)
|
||||
}
|
||||
|
||||
/// Parse markdown content with YAML front matter
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn parse_markdown_content(content: &str) -> Result<ParsedMarkdown> {
|
||||
@@ -97,5 +83,5 @@ pub(crate) fn parse_markdown_content(content: &str) -> Result<ParsedMarkdown> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "parser_test.rs"]
|
||||
#[path = "parser_tests.rs"]
|
||||
mod parser_test;
|
||||
|
||||
@@ -46,5 +46,5 @@ pub fn read_skills(path: &Path) -> Vec<ParsedSkill> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "read_skills_test.rs"]
|
||||
#[path = "read_skills_tests.rs"]
|
||||
mod read_skills_test;
|
||||
|
||||
+6
-3
@@ -1,6 +1,9 @@
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_read_skills_with_valid_skills() {
|
||||
@@ -49,7 +52,7 @@ This is the second test skill.
|
||||
let skill1 = skills.iter().find(|s| s.name == "test-skill-1").unwrap();
|
||||
assert_eq!(
|
||||
skill1.path,
|
||||
skill1_dir.join("SKILL.md").to_string_lossy().to_string()
|
||||
LocalOrRemotePath::Local(skill1_dir.join("SKILL.md"))
|
||||
);
|
||||
assert_eq!(skill1.description, "First test skill");
|
||||
assert!(skill1.content.contains("# Test Skill 1"));
|
||||
@@ -60,7 +63,7 @@ This is the second test skill.
|
||||
let skill2 = skills.iter().find(|s| s.name == "test-skill-2").unwrap();
|
||||
assert_eq!(
|
||||
skill2.path,
|
||||
skill2_dir.join("SKILL.md").to_string_lossy().to_string()
|
||||
LocalOrRemotePath::Local(skill2_dir.join("SKILL.md"))
|
||||
);
|
||||
assert_eq!(skill2.description, "Second test skill");
|
||||
assert!(skill2.content.contains("# Test Skill 2"));
|
||||
@@ -3,16 +3,16 @@
|
||||
//! This module defines the supported skill providers (i.e. Agents, Claude, Codex, Warp) and their
|
||||
//! associated skills directory paths. It provides utilities for looking up providers
|
||||
//! from paths and vice versa.
|
||||
use dirs::home_dir;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use dirs::home_dir;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use strum_macros::{Display, EnumString, VariantNames};
|
||||
use galaxy_core::ui::color::CLAUDE_ORANGE;
|
||||
use galaxy_core::ui::icons::Icon;
|
||||
use galaxy_core::ui::theme::Fill;
|
||||
use strum_macros::{Display, EnumString, VariantNames};
|
||||
use galaxy_util::local_or_remote_path::LocalOrRemotePath;
|
||||
|
||||
/// Represents a skill provider/origin (Agents, Claude, Codex, or Warp).
|
||||
#[derive(
|
||||
@@ -167,29 +167,65 @@ pub fn home_skills_path(provider: SkillProvider) -> Option<PathBuf> {
|
||||
home_dir().map(|home_dir| home_dir.join(&definition.skills_path))
|
||||
}
|
||||
|
||||
/// Returns the skill provider for a given path, if it matches a known skill provider directory.
|
||||
/// For example:
|
||||
/// get_provider_for_path(Path::new("/repo/.claude/skills/my-skill/SKILL.md")) returns Some(SkillProvider::Claude).
|
||||
/// Handles both SKILL.md files and files nested within a skill directory.
|
||||
pub fn get_provider_for_path(path: &Path) -> Option<SkillProvider> {
|
||||
let path_components: Vec<_> = path.components().collect();
|
||||
/// Returns the skill provider for a location, if it matches a known skill provider directory.
|
||||
///
|
||||
/// Local locations retain home-directory-aware matching. All other locations are
|
||||
/// classified by provider-directory structure using their standardized path representation.
|
||||
pub fn get_provider_for_path(path: &LocalOrRemotePath) -> Option<SkillProvider> {
|
||||
path.to_local_path()
|
||||
.and_then(get_home_provider_for_local_path)
|
||||
.or_else(|| get_provider_for_structural_path(path))
|
||||
}
|
||||
|
||||
for def in SKILL_PROVIDER_DEFINITIONS.iter() {
|
||||
if home_skills_path(def.provider)
|
||||
.into_iter()
|
||||
.any(|home_skills_path| path.starts_with(home_skills_path))
|
||||
{
|
||||
return Some(def.provider);
|
||||
fn get_home_provider_for_local_path(path: &Path) -> Option<SkillProvider> {
|
||||
SKILL_PROVIDER_DEFINITIONS
|
||||
.iter()
|
||||
.find(|definition| {
|
||||
home_skills_path(definition.provider)
|
||||
.into_iter()
|
||||
.any(|home_skills_path| path.starts_with(home_skills_path))
|
||||
})
|
||||
.map(|definition| definition.provider)
|
||||
}
|
||||
|
||||
/// Returns the directory containing a provider's skills root when `skills_root` has a known
|
||||
/// provider directory suffix, preserving the original local or remote location encoding.
|
||||
///
|
||||
/// For example, `/repo/.agents/skills` resolves to `/repo`, regardless of whether the location
|
||||
/// is encoded with Unix or Windows path separators.
|
||||
pub fn provider_parent_directory_for_skills_root(
|
||||
skills_root: &LocalOrRemotePath,
|
||||
) -> Option<LocalOrRemotePath> {
|
||||
match_provider_skills_root(skills_root).map(|(_, parent_directory)| parent_directory)
|
||||
}
|
||||
|
||||
fn get_provider_for_structural_path(path: &LocalOrRemotePath) -> Option<SkillProvider> {
|
||||
let mut current = Some(path.clone());
|
||||
while let Some(candidate) = current {
|
||||
if let Some((provider, _)) = match_provider_skills_root(&candidate) {
|
||||
return Some(provider);
|
||||
}
|
||||
current = candidate.parent();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// Retrieves path components for the skill provider directory (i.e., [".claude", "skills"])
|
||||
let skill_components: Vec<_> = def.skills_path.components().collect();
|
||||
|
||||
// Checks if some consecutive components of the path match the skill provider directory
|
||||
for window in path_components.windows(skill_components.len()) {
|
||||
if window == skill_components.as_slice() {
|
||||
return Some(def.provider);
|
||||
fn match_provider_skills_root(
|
||||
skills_root: &LocalOrRemotePath,
|
||||
) -> Option<(SkillProvider, LocalOrRemotePath)> {
|
||||
for definition in SKILL_PROVIDER_DEFINITIONS.iter() {
|
||||
let mut parent_directory = skills_root.clone();
|
||||
let mut matches_provider = true;
|
||||
for component in definition.skills_path.components().rev() {
|
||||
let expected_component = component.as_os_str().to_str()?;
|
||||
if parent_directory.file_name() != Some(expected_component) {
|
||||
matches_provider = false;
|
||||
break;
|
||||
}
|
||||
parent_directory = parent_directory.parent()?;
|
||||
}
|
||||
if matches_provider {
|
||||
return Some((definition.provider, parent_directory));
|
||||
}
|
||||
}
|
||||
None
|
||||
@@ -211,28 +247,5 @@ pub fn get_scope_for_path(path: &Path) -> SkillScope {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
get_provider_for_path, get_scope_for_path, home_skills_path, SkillProvider, SkillScope,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn warp_home_skills_path_uses_warp_home_path() {
|
||||
assert_eq!(
|
||||
home_skills_path(SkillProvider::Warp),
|
||||
galaxy_core::paths::galaxy_home_skills_dir()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warp_home_skill_path_is_home_warp_skill() {
|
||||
let Some(galaxy_home_skills_dir) = galaxy_core::paths::galaxy_home_skills_dir() else {
|
||||
eprintln!("Skipping test: home directory not available");
|
||||
return;
|
||||
};
|
||||
let path = galaxy_home_skills_dir.join("my-skill").join("SKILL.md");
|
||||
|
||||
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Warp));
|
||||
assert_eq!(get_scope_for_path(&path), SkillScope::Home);
|
||||
}
|
||||
}
|
||||
#[path = "skill_provider_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
use warp_util::host_id::HostId;
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
use warp_util::remote_path::RemotePath;
|
||||
use warp_util::standardized_path::StandardizedPath;
|
||||
|
||||
use super::{
|
||||
get_provider_for_path, get_scope_for_path, home_skills_path,
|
||||
provider_parent_directory_for_skills_root, SkillProvider, SkillScope,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn warp_home_skills_path_uses_warp_home_path() {
|
||||
assert_eq!(
|
||||
home_skills_path(SkillProvider::Warp),
|
||||
galaxy_core::paths::warp_home_skills_dir()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warp_home_skill_path_is_home_warp_skill() {
|
||||
let Some(warp_home_skills_dir) = galaxy_core::paths::warp_home_skills_dir() else {
|
||||
eprintln!("Skipping test: home directory not available");
|
||||
return;
|
||||
};
|
||||
let path = warp_home_skills_dir.join("my-skill").join("SKILL.md");
|
||||
|
||||
assert_eq!(
|
||||
get_provider_for_path(&LocalOrRemotePath::Local(path.clone())),
|
||||
Some(SkillProvider::Warp)
|
||||
);
|
||||
assert_eq!(get_scope_for_path(&path), SkillScope::Home);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_provider_path_is_classified_by_structure() {
|
||||
let path = LocalOrRemotePath::Remote(RemotePath::new(
|
||||
HostId::new("remote-host".to_string()),
|
||||
StandardizedPath::try_new("/repo/.claude/skills/my-skill/SKILL.md").unwrap(),
|
||||
));
|
||||
|
||||
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Claude));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_project_provider_path_is_classified_by_structure() {
|
||||
let path = LocalOrRemotePath::Local(
|
||||
std::env::temp_dir()
|
||||
.join("repo")
|
||||
.join(".claude")
|
||||
.join("skills")
|
||||
.join("my-skill")
|
||||
.join("SKILL.md"),
|
||||
);
|
||||
|
||||
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Claude));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreign_encoded_remote_provider_path_is_classified_by_structure() {
|
||||
let path = LocalOrRemotePath::Remote(RemotePath::new(
|
||||
HostId::new("remote-host".to_string()),
|
||||
StandardizedPath::try_new(r"C:\repo\.codex\skills\my-skill\SKILL.md").unwrap(),
|
||||
));
|
||||
|
||||
assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Codex));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreign_encoded_remote_skills_root_resolves_provider_parent_directory() {
|
||||
let host_id = HostId::new("remote-host".to_string());
|
||||
let skills_root = LocalOrRemotePath::Remote(RemotePath::new(
|
||||
host_id.clone(),
|
||||
StandardizedPath::try_new(r"C:\repo\.agents\skills").unwrap(),
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
provider_parent_directory_for_skills_root(&skills_root),
|
||||
Some(LocalOrRemotePath::Remote(RemotePath::new(
|
||||
host_id,
|
||||
StandardizedPath::try_new(r"C:\repo").unwrap(),
|
||||
)))
|
||||
);
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{fmt, path::PathBuf};
|
||||
use warp_util::local_or_remote_path::LocalOrRemotePath;
|
||||
|
||||
/// An unique reference to a skill.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
|
||||
pub enum SkillReference {
|
||||
/// A skill identified by the path to its SKILL.md file.
|
||||
Path(PathBuf),
|
||||
Path(LocalOrRemotePath),
|
||||
/// A bundled skill distributed with Warp.
|
||||
BundledSkillId(String),
|
||||
}
|
||||
@@ -13,7 +15,7 @@ pub enum SkillReference {
|
||||
impl fmt::Display for SkillReference {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
SkillReference::Path(path) => path.display().fmt(f),
|
||||
SkillReference::Path(path) => path.display_path().fmt(f),
|
||||
SkillReference::BundledSkillId(id) => write!(f, "@warp-skill:{id}"),
|
||||
}
|
||||
}
|
||||
@@ -23,9 +25,7 @@ impl From<SkillReference> for warp_multi_agent_api::skill_descriptor::SkillRefer
|
||||
fn from(reference: SkillReference) -> Self {
|
||||
match reference {
|
||||
SkillReference::Path(path) => {
|
||||
warp_multi_agent_api::skill_descriptor::SkillReference::Path(
|
||||
path.to_string_lossy().to_string(),
|
||||
)
|
||||
warp_multi_agent_api::skill_descriptor::SkillReference::Path(path.display_path())
|
||||
}
|
||||
SkillReference::BundledSkillId(id) => {
|
||||
warp_multi_agent_api::skill_descriptor::SkillReference::BundledSkillId(id)
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use galaxy_core::{
|
||||
features::FeatureFlag,
|
||||
register_telemetry_event,
|
||||
telemetry::{EnablementState, TelemetryEvent, TelemetryEventDesc},
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
use strum_macros::{EnumDiscriminants, EnumIter};
|
||||
use galaxy_core::features::FeatureFlag;
|
||||
use galaxy_core::register_telemetry_event;
|
||||
use galaxy_core::telemetry::{EnablementState, TelemetryEvent, TelemetryEventDesc};
|
||||
|
||||
#[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
|
||||
#[derive(Clone, EnumDiscriminants)]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use chrono::{DateTime, Days, Utc};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use chrono::{DateTime, Days, Utc};
|
||||
|
||||
/// Public-facing metadata persisted in SQLite
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct WorkspaceMetadata {
|
||||
|
||||
Reference in New Issue
Block a user