Initial public release of Warp.

Repo-Sync-Origin: warpdotdev/warp-internal@12af1d983b

crates/ai/Cargo.toml

@@ -0,0 +1,85 @@
+[package]
+name = "ai"
+version = "0.1.0"
+edition = "2021"
+publish.workspace = true
+license.workspace = true
+
+[features]
+jemalloc = []
+local_fs = []
+test-util = []
+crash_reporting = ["dep:sentry"]
+
+[dependencies]
+async-channel.workspace = true
+async-trait.workspace = true
+bincode.workspace = true
+bytes.workspace = true
+cfg-if.workspace = true
+computer_use.workspace = true
+dunce.workspace = true
+prost-types.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_yaml.workspace = true
+string-offset.workspace = true
+typed-path.workspace = true
+anyhow.workspace = true
+base16ct = "0.2.0"
+streaming-iterator.workspace = true
+instant.workspace = true
+itertools.workspace = true
+rayon.workspace = true
+ignore = "0.4.23"
+line-span = "0.1.5"
+log.workspace = true
+sentry = { workspace = true, optional = true }
+shellexpand.workspace = true
+strum.workspace = true
+strum_macros.workspace = true
+warpui.workspace = true
+warpui_extras = { workspace = true, features = ["default"] }
+warp_graphql.workspace = true
+warp_multi_agent_api.workspace = true
+dirs.workspace = true
+lazy_static.workspace = true
+regex.workspace = true
+rmcp.workspace = true
+sha2 = "0.10.8"
+strsim.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["rt"] }
+futures.workspace = true
+generic-array = "0.14.7"
+derivative.workspace = true
+warp_core.workspace = true
+warp_terminal.workspace = true
+warp_util.workspace = true
+chrono.workspace = true
+persistence.workspace = true
+priority-queue = "2.3.1"
+repo_metadata.workspace = true
+uuid.workspace = true
+unicode-width.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+nix.workspace = true
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+async-fs.workspace = true
+arborium.workspace = true
+languages.workspace = true
+syntax_tree.workspace = true
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+watcher.workspace = true
+notify-debouncer-full.workspace = true
+
+[dev-dependencies]
+filetime = "0.2.25"
+tempfile.workspace = true
+virtual-fs.workspace = true
+warp_core = { workspace = true, features = ["test-util"] }
+
+[build-dependencies]
+anyhow.workspace = true

crates/ai/build.rs

@@ -0,0 +1,11 @@
+use anyhow::Result;
+
+fn main() -> Result<()> {
+    let target_family = std::env::var("CARGO_CFG_TARGET_FAMILY")?;
+
+    if target_family != "wasm" {
+        println!("cargo:rustc-cfg=feature=\"local_fs\"");
+    }
+
+    Ok(())
+}

crates/ai/src/agent/action/convert.rs

@@ -0,0 +1,734 @@
+use std::{path::PathBuf, time::Duration};
+
+use itertools::Itertools as _;
+use uuid::Uuid;
+use warp_core::features::FeatureFlag;
+use warp_multi_agent_api as api;
+
+use crate::{
+    agent::{
+        action::{
+            AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
+            InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, ReadSkillRequest,
+            SearchCodebaseRequest, ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest,
+            UseComputerRequest,
+        },
+        action_result::{AnyFileContent, FileContext},
+        convert::ToolToAIAgentActionError,
+        FileLocations,
+    },
+    diff_validation::{ParsedDiff, V4AHunk},
+    document::AIDocumentId,
+    skills::SkillReference,
+};
+
+impl From<api::message::tool_call::RunShellCommand> for AIAgentActionType {
+    fn from(value: api::message::tool_call::RunShellCommand) -> Self {
+        AIAgentActionType::RequestCommandOutput {
+            command: value.command,
+            is_read_only: Some(value.is_read_only),
+            rationale: None,
+            uses_pager: Some(value.uses_pager),
+            is_risky: Some(value.is_risky),
+            wait_until_completion: value.wait_until_complete_value.is_none_or(
+                |api::message::tool_call::run_shell_command::WaitUntilCompleteValue::WaitUntilComplete(
+                    should_wait,
+                )| should_wait,
+            ),
+            citations: value
+                .citations
+                .iter()
+                .filter_map(|citation| citation.clone().try_into().ok())
+                .collect(),
+        }
+    }
+}
+
+impl From<api::message::tool_call::WriteToLongRunningShellCommand> for AIAgentActionType {
+    fn from(value: api::message::tool_call::WriteToLongRunningShellCommand) -> Self {
+        AIAgentActionType::WriteToLongRunningShellCommand {
+            block_id: value.command_id.into(),
+            input: value.input.into(),
+            mode: value.mode.map(Into::into).unwrap_or_default(),
+        }
+    }
+}
+
+impl From<api::message::tool_call::write_to_long_running_shell_command::Mode>
+    for AIAgentPtyWriteMode
+{
+    fn from(value: api::message::tool_call::write_to_long_running_shell_command::Mode) -> Self {
+        match value.mode {
+            Some(mode) => {
+                use warp_multi_agent_api::message::tool_call::write_to_long_running_shell_command::mode::Mode;
+                match mode {
+                    Mode::Raw(_) => AIAgentPtyWriteMode::Raw,
+                    Mode::Line(_) => AIAgentPtyWriteMode::Line,
+                    Mode::Block(_) => AIAgentPtyWriteMode::Block,
+                }
+            }
+            None => AIAgentPtyWriteMode::Raw,
+        }
+    }
+}
+
+impl From<api::message::tool_call::SuggestNewConversation> for AIAgentActionType {
+    fn from(value: api::message::tool_call::SuggestNewConversation) -> Self {
+        AIAgentActionType::SuggestNewConversation {
+            message_id: value.message_id,
+        }
+    }
+}
+
+impl From<api::message::tool_call::ApplyFileDiffs> for AIAgentActionType {
+    fn from(value: api::message::tool_call::ApplyFileDiffs) -> Self {
+        let diff_edits = value.diffs.into_iter().map(|file_diff| {
+            FileEdit::Edit(ParsedDiff::StrReplaceEdit {
+                search: file_diff.search.none_if_default(),
+                replace: file_diff.replace.none_if_default(),
+                file: file_diff.file_path.none_if_default(),
+            })
+        });
+        let v4a_updates = value.v4a_updates.into_iter().map(|v4a_update| {
+            FileEdit::Edit(ParsedDiff::V4AEdit {
+                file: v4a_update.file_path.none_if_default(),
+                move_to: v4a_update.move_to.clone().none_if_default(),
+                hunks: v4a_update
+                    .hunks
+                    .into_iter()
+                    .map(|hunk| V4AHunk {
+                        change_context: hunk.change_context,
+                        pre_context: hunk.pre_context,
+                        old: hunk.old,
+                        new: hunk.new,
+                        post_context: hunk.post_context,
+                    })
+                    .collect(),
+            })
+        });
+        let file_deletes = value
+            .deleted_files
+            .into_iter()
+            .map(|file_delete| FileEdit::Delete {
+                file: file_delete.file_path.none_if_default(),
+            });
+        let new_file_edits = value
+            .new_files
+            .into_iter()
+            .map(|new_file| FileEdit::Create {
+                file: new_file.file_path.none_if_default(),
+                content: new_file.content.none_if_default(),
+            });
+
+        AIAgentActionType::RequestFileEdits {
+            file_edits: diff_edits
+                .chain(v4a_updates)
+                .chain(new_file_edits)
+                .chain(file_deletes)
+                .collect(),
+            title: Some(value.summary),
+        }
+    }
+}
+
+impl From<api::message::tool_call::ReadFiles> for AIAgentActionType {
+    fn from(value: api::message::tool_call::ReadFiles) -> Self {
+        AIAgentActionType::ReadFiles(ReadFilesRequest {
+            locations: value.files.into_iter().map(Into::into).collect(),
+        })
+    }
+}
+
+impl TryFrom<api::UploadFileArtifact> for AIAgentActionType {
+    type Error = ToolToAIAgentActionError;
+
+    fn try_from(value: api::UploadFileArtifact) -> Result<Self, Self::Error> {
+        let file = value
+            .file
+            .filter(|file| !file.file_path.is_empty())
+            .ok_or(ToolToAIAgentActionError::MissingUploadArtifactFileReference)?;
+
+        Ok(AIAgentActionType::UploadArtifact(UploadArtifactRequest {
+            file_path: file.file_path,
+            description: value.description.none_if_default(),
+        }))
+    }
+}
+
+impl From<api::message::tool_call::SearchCodebase> for AIAgentActionType {
+    fn from(value: api::message::tool_call::SearchCodebase) -> Self {
+        AIAgentActionType::SearchCodebase(SearchCodebaseRequest {
+            query: value.query,
+            partial_paths: if !value.path_filters.is_empty() {
+                Some(value.path_filters)
+            } else {
+                None
+            },
+            codebase_path: if !value.codebase_path.is_empty() {
+                Some(value.codebase_path)
+            } else {
+                None
+            },
+        })
+    }
+}
+
+impl From<api::message::tool_call::Grep> for AIAgentActionType {
+    fn from(value: api::message::tool_call::Grep) -> Self {
+        AIAgentActionType::Grep {
+            queries: value.queries,
+            path: value.path,
+        }
+    }
+}
+
+impl From<api::message::tool_call::FileGlob> for AIAgentActionType {
+    fn from(value: api::message::tool_call::FileGlob) -> Self {
+        AIAgentActionType::FileGlob {
+            patterns: value.patterns,
+            path: if value.path.is_empty() {
+                None
+            } else {
+                Some(value.path)
+            },
+        }
+    }
+}
+
+impl From<api::message::tool_call::FileGlobV2> for AIAgentActionType {
+    fn from(value: api::message::tool_call::FileGlobV2) -> Self {
+        AIAgentActionType::FileGlobV2 {
+            patterns: value.patterns,
+            search_dir: if value.search_dir.is_empty() {
+                None
+            } else {
+                Some(value.search_dir)
+            },
+        }
+    }
+}
+
+impl From<api::message::tool_call::read_files::File> for FileLocations {
+    fn from(value: api::message::tool_call::read_files::File) -> Self {
+        Self {
+            name: value.name,
+            lines: value
+                .line_ranges
+                .into_iter()
+                .map(|line_range| line_range.start as usize..line_range.end as usize)
+                .collect(),
+        }
+    }
+}
+
+impl From<api::message::tool_call::ReadMcpResource> for AIAgentActionType {
+    fn from(value: api::message::tool_call::ReadMcpResource) -> Self {
+        let server_id = if FeatureFlag::MCPGroupedServerContext.is_enabled() {
+            Uuid::parse_str(&value.server_id).ok()
+        } else {
+            None
+        };
+
+        AIAgentActionType::ReadMCPResource {
+            server_id,
+            uri: Some(value.uri),
+            name: Default::default(),
+        }
+    }
+}
+
+impl TryFrom<api::message::tool_call::CallMcpTool> for AIAgentActionType {
+    type Error = ToolToAIAgentActionError;
+
+    fn try_from(value: api::message::tool_call::CallMcpTool) -> Result<Self, Self::Error> {
+        let Some(args) = value.args else {
+            return Err(ToolToAIAgentActionError::CallMCPToolArgsError(
+                String::from("missing args"),
+            ));
+        };
+
+        let input = prost_to_serde_json(prost_types::Value {
+            kind: Some(prost_types::value::Kind::StructValue(args)),
+        })
+        .map_err(ToolToAIAgentActionError::CallMCPToolArgsError)?;
+
+        let server_id = if FeatureFlag::MCPGroupedServerContext.is_enabled() {
+            Uuid::parse_str(&value.server_id).ok()
+        } else {
+            None
+        };
+
+        Ok(AIAgentActionType::CallMCPTool {
+            server_id,
+            name: value.name,
+            input,
+        })
+    }
+}
+
+impl TryFrom<api::message::tool_call::SuggestPrompt> for AIAgentActionType {
+    type Error = ToolToAIAgentActionError;
+
+    fn try_from(value: api::message::tool_call::SuggestPrompt) -> Result<Self, Self::Error> {
+        let request = match value.display_mode {
+            Some(api::message::tool_call::suggest_prompt::DisplayMode::InlineQueryBanner(
+                inline_query_banner,
+            )) => SuggestPromptRequest::UnitTestsSuggestion {
+                title: inline_query_banner.title,
+                description: inline_query_banner.description,
+                query: inline_query_banner.query,
+            },
+            Some(api::message::tool_call::suggest_prompt::DisplayMode::PromptChip(chip)) => {
+                let label = chip.label.none_if_default();
+                SuggestPromptRequest::PromptSuggestion {
+                    prompt: chip.prompt,
+                    label,
+                }
+            }
+            _ => {
+                return Err(ToolToAIAgentActionError::SuggestPromptError(String::from(
+                    "unsupported display mode",
+                )));
+            }
+        };
+        Ok(AIAgentActionType::SuggestPrompt(request))
+    }
+}
+
+impl From<warp_multi_agent_api::FileContent> for FileContext {
+    fn from(content: warp_multi_agent_api::FileContent) -> Self {
+        let line_range = content.line_range.map(|r| r.start as usize..r.end as usize);
+
+        FileContext::new(
+            content.file_path,
+            AnyFileContent::StringContent(content.content),
+            line_range,
+            None,
+        )
+    }
+}
+
+impl From<warp_multi_agent_api::AnyFileContent> for FileContext {
+    fn from(content: warp_multi_agent_api::AnyFileContent) -> Self {
+        match content.content {
+            Some(api::any_file_content::Content::BinaryContent(binary_content)) => {
+                FileContext::new(
+                    binary_content.file_path,
+                    AnyFileContent::BinaryContent(binary_content.data),
+                    None,
+                    None,
+                )
+            }
+            Some(api::any_file_content::Content::TextContent(text_content)) => {
+                let line_range = text_content
+                    .line_range
+                    .map(|r| r.start as usize..r.end as usize);
+
+                FileContext::new(
+                    text_content.file_path,
+                    AnyFileContent::StringContent(text_content.content),
+                    line_range,
+                    None,
+                )
+            }
+            None => unreachable!("AnyFileContent should always have a content"),
+        }
+    }
+}
+
+impl From<api::message::tool_call::ReadDocuments> for AIAgentActionType {
+    fn from(value: api::message::tool_call::ReadDocuments) -> Self {
+        use crate::agent::action::ReadDocumentsRequest;
+        AIAgentActionType::ReadDocuments(ReadDocumentsRequest {
+            document_ids: value
+                .documents
+                .into_iter()
+                .filter_map(|doc| AIDocumentId::try_from(doc.document_id).ok())
+                .collect(),
+        })
+    }
+}
+
+impl From<api::message::tool_call::EditDocuments> for AIAgentActionType {
+    fn from(value: api::message::tool_call::EditDocuments) -> Self {
+        use crate::agent::action::{DocumentDiff, EditDocumentsRequest};
+        AIAgentActionType::EditDocuments(EditDocumentsRequest {
+            diffs: value
+                .diffs
+                .into_iter()
+                .filter_map(|diff| {
+                    AIDocumentId::try_from(diff.document_id)
+                        .map(|document_id| DocumentDiff {
+                            document_id,
+                            search: diff.search,
+                            replace: diff.replace,
+                        })
+                        .ok()
+                })
+                .collect(),
+        })
+    }
+}
+
+impl From<api::message::tool_call::CreateDocuments> for AIAgentActionType {
+    fn from(value: api::message::tool_call::CreateDocuments) -> Self {
+        use crate::agent::action::{CreateDocumentsRequest, DocumentToCreate};
+        AIAgentActionType::CreateDocuments(CreateDocumentsRequest {
+            documents: value
+                .new_documents
+                .into_iter()
+                .map(|doc| DocumentToCreate {
+                    content: doc.content,
+                    title: if doc.title.is_empty() {
+                        // DO NOT SUBMIT
+                        // crate::ai::ai_document_view::DEFAULT_PLANNING_DOCUMENT_TITLE.to_string()
+                        "".to_string()
+                    } else {
+                        doc.title
+                    },
+                })
+                .collect(),
+        })
+    }
+}
+
+impl From<api::message::tool_call::ReadShellCommandOutput> for AIAgentActionType {
+    fn from(value: api::message::tool_call::ReadShellCommandOutput) -> Self {
+        let delay = match value.delay {
+            Some(api::message::tool_call::read_shell_command_output::Delay::Duration(duration)) => {
+                Some(ShellCommandDelay::Duration(Duration::from_secs(
+                    duration.seconds as u64,
+                )))
+            }
+            Some(api::message::tool_call::read_shell_command_output::Delay::OnCompletion(_)) => {
+                Some(ShellCommandDelay::OnCompletion)
+            }
+            None => None,
+        };
+        AIAgentActionType::ReadShellCommandOutput {
+            block_id: value.command_id.into(),
+            delay,
+        }
+    }
+}
+
+impl From<api::message::tool_call::TransferShellCommandControlToUser> for AIAgentActionType {
+    fn from(value: api::message::tool_call::TransferShellCommandControlToUser) -> Self {
+        AIAgentActionType::TransferShellCommandControlToUser {
+            reason: value.reason,
+        }
+    }
+}
+
+impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
+    type Error = ToolToAIAgentActionError;
+
+    fn try_from(value: api::message::tool_call::UseComputer) -> Result<Self, Self::Error> {
+        use api::message::tool_call::use_computer;
+
+        let actions = value
+            .actions
+            .into_iter()
+            .map(|action| {
+                let Some(action_type) = action.r#type else {
+                    return Err(ToolToAIAgentActionError::MissingComputerUseActionType);
+                };
+                match action_type {
+                    use_computer::action::Type::MouseMove(mouse_move) => {
+                        Ok(computer_use::Action::MouseMove {
+                            to: coordinates_to_vec(mouse_move.to.as_ref())?,
+                        })
+                    }
+                    use_computer::action::Type::MouseDown(mouse_down) => {
+                        Ok(computer_use::Action::MouseDown {
+                            button: to_computer_use_button(mouse_down.button()),
+                            at: coordinates_to_vec(mouse_down.at.as_ref())?,
+                        })
+                    }
+                    use_computer::action::Type::MouseUp(mouse_up) => {
+                        Ok(computer_use::Action::MouseUp {
+                            button: to_computer_use_button(mouse_up.button()),
+                        })
+                    }
+                    use_computer::action::Type::MouseWheel(mouse_wheel) => {
+                        let direction = to_scroll_direction(mouse_wheel.direction());
+                        let distance = to_scroll_distance(mouse_wheel.distance)?;
+                        Ok(computer_use::Action::MouseWheel {
+                            at: coordinates_to_vec(mouse_wheel.at.as_ref())?,
+                            direction,
+                            distance,
+                        })
+                    }
+                    use_computer::action::Type::Wait(wait) => {
+                        let duration = wait.duration.unwrap_or_default();
+                        if duration.seconds < 0 || duration.nanos < 0 {
+                            return Err(ToolToAIAgentActionError::InvalidComputerUseWaitDuration);
+                        }
+                        let duration = Duration::from_secs(duration.seconds as u64)
+                            + Duration::from_nanos(duration.nanos as u64);
+                        Ok(computer_use::Action::Wait(duration))
+                    }
+                    use_computer::action::Type::TypeText(type_text) => {
+                        Ok(computer_use::Action::TypeText {
+                            text: type_text.text,
+                        })
+                    }
+                    use_computer::action::Type::KeyDown(key_down) => {
+                        let key = convert_key(key_down.key)?;
+                        Ok(computer_use::Action::KeyDown { key })
+                    }
+                    use_computer::action::Type::KeyUp(key_up) => {
+                        let key = convert_key(key_up.key)?;
+                        Ok(computer_use::Action::KeyUp { key })
+                    }
+                }
+            })
+            .try_collect()?;
+        let screenshot_params = value
+            .post_actions_screenshot_params
+            .map(convert_screenshot_params);
+        Ok(AIAgentActionType::UseComputer(UseComputerRequest {
+            action_summary: value.action_summary,
+            actions,
+            screenshot_params,
+        }))
+    }
+}
+
+impl From<api::message::tool_call::RequestComputerUse> for AIAgentActionType {
+    fn from(value: api::message::tool_call::RequestComputerUse) -> Self {
+        use crate::agent::action::RequestComputerUseRequest;
+        AIAgentActionType::RequestComputerUse(RequestComputerUseRequest {
+            task_summary: value.task_summary,
+            screenshot_params: value.screenshot_params.map(convert_screenshot_params),
+        })
+    }
+}
+
+impl TryFrom<api::message::tool_call::ReadSkill> for AIAgentActionType {
+    type Error = ToolToAIAgentActionError;
+
+    fn try_from(value: api::message::tool_call::ReadSkill) -> Result<Self, Self::Error> {
+        match value.skill_reference {
+            Some(reference) => Ok(AIAgentActionType::ReadSkill(ReadSkillRequest {
+                skill: SkillReference::from(reference),
+            })),
+            None => Err(ToolToAIAgentActionError::MissingSkillReference),
+        }
+    }
+}
+
+impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
+    fn from(value: api::message::tool_call::FetchConversation) -> Self {
+        AIAgentActionType::FetchConversation {
+            conversation_id: value.conversation_id,
+        }
+    }
+}
+
+impl From<api::message::tool_call::read_skill::SkillReference> for SkillReference {
+    fn from(value: api::message::tool_call::read_skill::SkillReference) -> Self {
+        use warp_multi_agent_api::message::tool_call::read_skill::SkillReference as ApiSkillReference;
+        match value {
+            ApiSkillReference::SkillPath(skill_path) => {
+                SkillReference::Path(PathBuf::from(skill_path))
+            }
+            ApiSkillReference::BundledSkillId(id) => SkillReference::BundledSkillId(id),
+        }
+    }
+}
+
+/// Converts API ScreenshotParams to the internal computer_use type.
+fn convert_screenshot_params(
+    params: api::message::tool_call::ScreenshotParams,
+) -> computer_use::ScreenshotParams {
+    let region = params
+        .region
+        .and_then(|r| match (r.top_left.as_ref(), r.bottom_right.as_ref()) {
+            (Some(tl), Some(br)) => Some(computer_use::ScreenshotRegion {
+                top_left: computer_use::Vector2I::new(tl.x, tl.y),
+                bottom_right: computer_use::Vector2I::new(br.x, br.y),
+            }),
+            _ => None,
+        });
+
+    computer_use::ScreenshotParams {
+        max_long_edge_px: (params.max_long_edge_px > 0).then_some(params.max_long_edge_px as usize),
+        max_total_px: (params.max_total_px > 0).then_some(params.max_total_px as usize),
+        region,
+    }
+}
+
+fn coordinates_to_vec(
+    coords: Option<&api::Coordinates>,
+) -> Result<computer_use::Vector2I, ToolToAIAgentActionError> {
+    match coords {
+        Some(coords) => Ok(computer_use::Vector2I::new(coords.x, coords.y)),
+        None => Err(ToolToAIAgentActionError::MissingComputerUseCoordinates),
+    }
+}
+
+fn to_computer_use_button(
+    api_button: api::message::tool_call::use_computer::action::MouseButton,
+) -> computer_use::MouseButton {
+    use api::message::tool_call::use_computer::action::MouseButton;
+    match api_button {
+        MouseButton::Left => computer_use::MouseButton::Left,
+        MouseButton::Right => computer_use::MouseButton::Right,
+        MouseButton::Middle => computer_use::MouseButton::Middle,
+        MouseButton::Back => computer_use::MouseButton::Back,
+        MouseButton::Forward => computer_use::MouseButton::Forward,
+    }
+}
+
+fn to_scroll_direction(
+    api_direction: api::message::tool_call::use_computer::action::mouse_wheel::Direction,
+) -> computer_use::ScrollDirection {
+    use api::message::tool_call::use_computer::action::mouse_wheel::Direction;
+    match api_direction {
+        Direction::Up => computer_use::ScrollDirection::Up,
+        Direction::Down => computer_use::ScrollDirection::Down,
+        Direction::Left => computer_use::ScrollDirection::Left,
+        Direction::Right => computer_use::ScrollDirection::Right,
+    }
+}
+
+fn to_scroll_distance(
+    api_distance: Option<api::message::tool_call::use_computer::action::mouse_wheel::Distance>,
+) -> Result<computer_use::ScrollDistance, ToolToAIAgentActionError> {
+    use api::message::tool_call::use_computer::action::mouse_wheel::Distance;
+    match api_distance {
+        Some(Distance::Pixels(pixels)) => Ok(computer_use::ScrollDistance::Pixels(pixels)),
+        Some(Distance::Clicks(clicks)) => Ok(computer_use::ScrollDistance::Clicks(clicks)),
+        None => Err(ToolToAIAgentActionError::MissingComputerUseScrollDistance),
+    }
+}
+
+fn convert_key(
+    api_key: Option<api::message::tool_call::use_computer::action::Key>,
+) -> Result<computer_use::Key, ToolToAIAgentActionError> {
+    use api::message::tool_call::use_computer::action::key::Data;
+    let key = api_key.ok_or(ToolToAIAgentActionError::MissingComputerUseKey)?;
+    match key.data {
+        Some(Data::Keycode(keycode)) => Ok(computer_use::Key::Keycode(keycode)),
+        Some(Data::Char(char_str)) => {
+            let mut chars = char_str.chars();
+            let ch = chars
+                .next()
+                .ok_or(ToolToAIAgentActionError::InvalidComputerUseCharKey)?;
+            if chars.next().is_some() {
+                return Err(ToolToAIAgentActionError::InvalidComputerUseCharKey);
+            }
+            Ok(computer_use::Key::Char(ch))
+        }
+        None => Err(ToolToAIAgentActionError::MissingComputerUseKey),
+    }
+}
+
+fn prost_to_serde_json(x: prost_types::Value) -> Result<serde_json::Value, String> {
+    use prost_types::value::Kind::*;
+    use serde_json::Value::*;
+
+    let Some(kind) = x.kind else {
+        return Err("google.protobuf.Value kind was None".to_string());
+    };
+
+    Ok(match kind {
+        NullValue(_) => Null,
+        BoolValue(v) => Bool(v),
+        NumberValue(n) => Number(
+            serde_json::Number::from_f64(n)
+                .ok_or_else(|| format!("float {n} is not valid JSON number"))?,
+        ),
+        StringValue(s) => String(s),
+        ListValue(l) => Array(
+            l.values
+                .into_iter()
+                .map(prost_to_serde_json)
+                .collect::<Result<Vec<_>, std::string::String>>()?,
+        ),
+        StructValue(v) => Object(
+            v.fields
+                .into_iter()
+                .map(|(k, v)| prost_to_serde_json(v).map(|v| (k, v)))
+                .collect::<Result<serde_json::Map<_, _>, std::string::String>>()?,
+        ),
+    })
+}
+
+/// Helper trait to easily convert default values to `None`.
+/// With `prost`, scalar types are converted to their default values instead
+/// of `None` if the type is unset. This trait allows a more natural
+/// conversion to better denote if a value should be `Some` (indicating it was set)
+/// or `None` (indicating it was unset).
+///
+///
+/// NOTE: Consumers should use this with caution as it only makes sense where the default
+/// value of the type isn't a reasonable value and `None` makes more sense instead.
+trait NoneIfDefault
+where
+    Self: Sized + Default,
+{
+    fn none_if_default(self) -> Option<Self>;
+}
+
+impl NoneIfDefault for String {
+    fn none_if_default(self) -> Option<Self> {
+        if self == Self::default() {
+            None
+        } else {
+            Some(self)
+        }
+    }
+}
+
+impl From<api::message::tool_call::InsertReviewComments> for AIAgentActionType {
+    fn from(value: api::message::tool_call::InsertReviewComments) -> Self {
+        AIAgentActionType::InsertCodeReviewComments {
+            repo_path: PathBuf::from(value.repo_path),
+            comments: value.comments.into_iter().map(Into::into).collect(),
+            base_branch: value.base_branch.none_if_default(),
+        }
+    }
+}
+
+impl From<api::message::tool_call::insert_review_comments::CommentSide> for CommentSide {
+    fn from(value: api::message::tool_call::insert_review_comments::CommentSide) -> Self {
+        match value {
+            api::message::tool_call::insert_review_comments::CommentSide::New => CommentSide::Right,
+            api::message::tool_call::insert_review_comments::CommentSide::Old => CommentSide::Left,
+        }
+    }
+}
+
+impl From<api::message::tool_call::insert_review_comments::Comment> for InsertReviewComment {
+    fn from(value: api::message::tool_call::insert_review_comments::Comment) -> Self {
+        let location = value.location.map(|loc| InsertedCommentLocation {
+            relative_file_path: loc.file_path,
+            line: loc.line.and_then(|comment_line_range| {
+                let side = comment_line_range.side().into();
+                let diff_hunk = comment_line_range.diff_hunk;
+                comment_line_range.range.map(|r| InsertedCommentLine {
+                    comment_line_range: r.start as usize..r.end as usize,
+                    diff_hunk_line_range: r.start as usize..r.end as usize,
+                    diff_hunk_text: diff_hunk,
+                    side: Some(side),
+                })
+            }),
+        });
+
+        InsertReviewComment {
+            comment_id: value.comment_id,
+            author: value.author,
+            last_modified_timestamp: value.last_modified_timestamp,
+            comment_body: value.comment_body,
+            parent_comment_id: if value.parent_comment_id.is_empty() {
+                None
+            } else {
+                Some(value.parent_comment_id)
+            },
+            comment_location: location,
+            html_url: value.html_url.none_if_default(),
+        }
+    }
+}

crates/ai/src/agent/action/mod.rs

@@ -0,0 +1,826 @@
+mod convert;
+
+use std::{fmt::Display, ops::Range, path::PathBuf, time::Duration};
+
+use itertools::Itertools as _;
+use serde::{Deserialize, Serialize};
+use strum_macros::EnumDiscriminants;
+use uuid::Uuid;
+use warp_terminal::model::BlockId;
+
+use crate::{
+    agent::{
+        action_result::{
+            AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult,
+            CreateDocumentsResult, EditDocumentsResult, FetchConversationResult, FileGlobResult,
+            FileGlobV2Result, GrepResult, InsertReviewCommentsResult, ReadDocumentsResult,
+            ReadFilesResult, ReadMCPResourceResult, ReadShellCommandOutputResult, ReadSkillResult,
+            RequestCommandOutputResult, RequestComputerUseResult, RequestFileEditsResult,
+            SearchCodebaseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
+            SuggestNewConversationResult, SuggestPromptResult,
+            TransferShellCommandControlToUserResult, UploadArtifactResult, UseComputerResult,
+            WriteToLongRunningShellCommandResult,
+        },
+        AIAgentCitation, FileLocations,
+    },
+    diff_validation::ParsedDiff,
+    document::AIDocumentId,
+    skills::SkillReference,
+};
+pub use warp_multi_agent_api::LifecycleEventType;
+
+#[derive(Debug, Clone, Eq, PartialEq, EnumDiscriminants)]
+pub enum AIAgentActionType {
+    /// The AI requested the output for a given command to be retrieved as context in responding to
+    /// a user's query.
+    RequestCommandOutput {
+        command: String,
+
+        /// [`Some(true)`] iff the LLM thinks that the `command` is readonly and doesn't produce side-effects.
+        is_read_only: Option<bool>,
+
+        /// [`Some(true)`] iff the LLM thinks that the `command` is risky and should require user confirmation.
+        is_risky: Option<bool>,
+
+        /// `true` if the client should wait until the command is completed and report the finish output as the result.
+        ///
+        /// If `false` _and_ the command is long-running, a snapshot of the command output is taken and reported as the
+        /// result instead.
+        wait_until_completion: bool,
+
+        /// [`Some(true)`] iff the LLM thinks that the `command` might invoke pager.
+        uses_pager: Option<bool>,
+
+        /// The AI's rationale for requesting a command.
+        rationale: Option<String>,
+
+        /// The citations for the command.
+        citations: Vec<AIAgentCitation>,
+    },
+
+    WriteToLongRunningShellCommand {
+        block_id: BlockId,
+        input: bytes::Bytes,
+        mode: AIAgentPtyWriteMode,
+    },
+
+    /// AI requested getting the content of some files.
+    ReadFiles(ReadFilesRequest),
+
+    /// AI requested uploading a local file as a conversation artifact.
+    UploadArtifact(UploadArtifactRequest),
+
+    SearchCodebase(SearchCodebaseRequest),
+
+    /// AI requested a vector of edits. Each edit holds a list of diffs on a single code file.
+    RequestFileEdits {
+        file_edits: Vec<FileEdit>,
+        title: Option<String>,
+    },
+
+    Grep {
+        queries: Vec<String>,
+        path: String,
+    },
+
+    FileGlob {
+        patterns: Vec<String>,
+        path: Option<String>,
+    },
+
+    FileGlobV2 {
+        patterns: Vec<String>,
+        search_dir: Option<String>,
+        // TODO(matthew): Maybe implement client side depth and result limits.
+    },
+
+    ReadMCPResource {
+        server_id: Option<Uuid>,
+        name: String,
+        /// The unique URI for the resource. Prefer using this to identify
+        /// a resource over [`ReadMCPResource::name`], when available.
+        ///
+        /// We should phase out `name` eventually and make this non-optional.
+        uri: Option<String>,
+    },
+
+    CallMCPTool {
+        server_id: Option<Uuid>,
+        name: String,
+        input: serde_json::Value,
+    },
+
+    SuggestNewConversation {
+        message_id: String,
+    },
+
+    SuggestPrompt(SuggestPromptRequest),
+
+    InitProject,
+    OpenCodeReview,
+
+    ReadDocuments(ReadDocumentsRequest),
+    EditDocuments(EditDocumentsRequest),
+    CreateDocuments(CreateDocumentsRequest),
+
+    ReadShellCommandOutput {
+        block_id: BlockId,
+        delay: Option<ShellCommandDelay>,
+    },
+
+    UseComputer(UseComputerRequest),
+
+    InsertCodeReviewComments {
+        repo_path: PathBuf,
+        comments: Vec<InsertReviewComment>,
+        base_branch: Option<String>,
+    },
+
+    RequestComputerUse(RequestComputerUseRequest),
+
+    // AI requested to read a skill.
+    ReadSkill(ReadSkillRequest),
+
+    FetchConversation {
+        conversation_id: String,
+    },
+
+    StartAgent {
+        version: StartAgentVersion,
+        name: String,
+        prompt: String,
+        execution_mode: StartAgentExecutionMode,
+        lifecycle_subscription: Option<Vec<LifecycleEventType>>,
+    },
+
+    SendMessageToAgent {
+        addresses: Vec<String>,
+        subject: String,
+        message: String,
+    },
+    /// Transfer control of a running shell command to the user.
+    TransferShellCommandControlToUser {
+        /// The reason provided by the agent for transferring control.
+        reason: String,
+    },
+
+    AskUserQuestion {
+        questions: Vec<AskUserQuestionItem>,
+    },
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum StartAgentExecutionMode {
+    Local {
+        /// `None` selects the legacy embedded local child-agent flow.
+        /// `Some(...)` selects a third-party CLI harness to launch locally.
+        harness_type: Option<String>,
+    },
+    Remote {
+        environment_id: String,
+        skill_references: Vec<SkillReference>,
+        model_id: String,
+        computer_use_enabled: bool,
+        worker_host: String,
+        harness_type: String,
+        title: String,
+    },
+}
+
+impl StartAgentExecutionMode {
+    /// Constructs a local execution mode using the legacy v1 default harness.
+    pub fn local_with_defaults() -> Self {
+        Self::Local { harness_type: None }
+    }
+    /// Constructs a local execution mode for a specific third-party harness.
+    pub fn local_harness(harness_type: String) -> Self {
+        Self::Local {
+            harness_type: Some(harness_type),
+        }
+    }
+    /// Constructs a remote execution mode using the legacy v1 defaults for
+    /// fields that were added later in StartAgentV2.
+    pub fn remote_with_defaults(environment_id: String) -> Self {
+        Self::Remote {
+            environment_id,
+            skill_references: Vec::new(),
+            model_id: String::new(),
+            computer_use_enabled: false,
+            worker_host: String::new(),
+            harness_type: String::new(),
+            title: String::new(),
+        }
+    }
+}
+impl AIAgentActionType {
+    pub fn is_request_command_output(&self) -> bool {
+        matches!(self, Self::RequestCommandOutput { .. })
+    }
+
+    pub fn is_read_files(&self) -> bool {
+        matches!(self, Self::ReadFiles(..))
+    }
+
+    pub fn is_search_codebase(&self) -> bool {
+        matches!(self, Self::SearchCodebase(..))
+    }
+
+    pub fn is_grep(&self) -> bool {
+        matches!(self, Self::Grep { .. })
+    }
+
+    pub fn is_file_glob(&self) -> bool {
+        matches!(self, Self::FileGlob { .. } | Self::FileGlobV2 { .. })
+    }
+
+    pub fn is_write_to_shell_command(&self) -> bool {
+        matches!(self, Self::WriteToLongRunningShellCommand { .. })
+    }
+
+    pub fn cancelled_result(&self) -> AIAgentActionResultType {
+        match self {
+            Self::RequestCommandOutput { .. } => AIAgentActionResultType::RequestCommandOutput(
+                RequestCommandOutputResult::CancelledBeforeExecution,
+            ),
+            Self::RequestFileEdits { .. } => {
+                AIAgentActionResultType::RequestFileEdits(RequestFileEditsResult::Cancelled)
+            }
+            Self::ReadFiles(..) => AIAgentActionResultType::ReadFiles(ReadFilesResult::Cancelled),
+            Self::UploadArtifact(..) => {
+                AIAgentActionResultType::UploadArtifact(UploadArtifactResult::Cancelled)
+            }
+            Self::SearchCodebase(..) => {
+                AIAgentActionResultType::SearchCodebase(SearchCodebaseResult::Cancelled)
+            }
+            Self::Grep { .. } => AIAgentActionResultType::Grep(GrepResult::Cancelled),
+            Self::FileGlob { .. } => AIAgentActionResultType::FileGlob(FileGlobResult::Cancelled),
+            Self::FileGlobV2 { .. } => {
+                AIAgentActionResultType::FileGlobV2(FileGlobV2Result::Cancelled)
+            }
+            Self::WriteToLongRunningShellCommand { .. } => {
+                AIAgentActionResultType::WriteToLongRunningShellCommand(
+                    WriteToLongRunningShellCommandResult::Cancelled,
+                )
+            }
+            Self::CallMCPTool { .. } => {
+                AIAgentActionResultType::CallMCPTool(CallMCPToolResult::Cancelled)
+            }
+            Self::ReadMCPResource { .. } => {
+                AIAgentActionResultType::ReadMCPResource(ReadMCPResourceResult::Cancelled)
+            }
+            Self::SuggestNewConversation { .. } => AIAgentActionResultType::SuggestNewConversation(
+                SuggestNewConversationResult::Cancelled,
+            ),
+            Self::SuggestPrompt { .. } => {
+                AIAgentActionResultType::SuggestPrompt(SuggestPromptResult::Cancelled)
+            }
+            Self::OpenCodeReview => AIAgentActionResultType::OpenCodeReview,
+            Self::InitProject => AIAgentActionResultType::InitProject,
+            Self::ReadDocuments(_) => {
+                AIAgentActionResultType::ReadDocuments(ReadDocumentsResult::Cancelled)
+            }
+            Self::EditDocuments(_) => {
+                AIAgentActionResultType::EditDocuments(EditDocumentsResult::Cancelled)
+            }
+            Self::CreateDocuments(_) => {
+                AIAgentActionResultType::CreateDocuments(CreateDocumentsResult::Cancelled)
+            }
+            Self::ReadShellCommandOutput { .. } => AIAgentActionResultType::ReadShellCommandOutput(
+                ReadShellCommandOutputResult::Cancelled,
+            ),
+            Self::UseComputer(_) => {
+                AIAgentActionResultType::UseComputer(UseComputerResult::Cancelled)
+            }
+            Self::InsertCodeReviewComments { .. } => {
+                AIAgentActionResultType::InsertReviewComments(InsertReviewCommentsResult::Cancelled)
+            }
+            Self::RequestComputerUse(_) => {
+                AIAgentActionResultType::RequestComputerUse(RequestComputerUseResult::Cancelled)
+            }
+            Self::ReadSkill(_) => AIAgentActionResultType::ReadSkill(ReadSkillResult::Cancelled),
+            Self::FetchConversation { .. } => {
+                AIAgentActionResultType::FetchConversation(FetchConversationResult::Cancelled)
+            }
+            Self::StartAgent { version, .. } => {
+                AIAgentActionResultType::StartAgent(StartAgentResult::Cancelled {
+                    version: *version,
+                })
+            }
+            Self::SendMessageToAgent { .. } => {
+                AIAgentActionResultType::SendMessageToAgent(SendMessageToAgentResult::Cancelled)
+            }
+            Self::TransferShellCommandControlToUser { .. } => {
+                AIAgentActionResultType::TransferShellCommandControlToUser(
+                    TransferShellCommandControlToUserResult::Cancelled,
+                )
+            }
+            Self::AskUserQuestion { .. } => {
+                AIAgentActionResultType::AskUserQuestion(AskUserQuestionResult::Cancelled)
+            }
+        }
+    }
+
+    pub fn user_friendly_name(&self) -> String {
+        match self {
+            Self::RequestCommandOutput { command, .. } => {
+                format!("Run command: {command}")
+            }
+            Self::WriteToLongRunningShellCommand { .. } => {
+                "Write to long running shell command".to_string()
+            }
+            Self::ReadFiles(_) => "Read files".to_string(),
+            Self::UploadArtifact(_) => "Upload artifact".to_string(),
+            Self::SearchCodebase(_) => "Search codebase".to_string(),
+            Self::RequestFileEdits { file_edits, .. } => {
+                let file_names = file_edits.iter().filter_map(|edit| edit.file()).join(", ");
+                format!("Edit {file_names}")
+            }
+            Self::Grep { .. } => "Grep".to_string(),
+            Self::FileGlob { .. } | Self::FileGlobV2 { .. } => "File glob".to_string(),
+            Self::ReadMCPResource { .. } => "Read mcp resource".to_string(),
+            Self::CallMCPTool { .. } => "Call mcp tool".to_string(),
+            Self::SuggestNewConversation { .. } => "Suggest new conversation".to_string(),
+            Self::SuggestPrompt { .. } => "Suggest prompt".to_string(),
+            Self::InitProject => "Init project".to_string(),
+            Self::OpenCodeReview => "Open code review".to_string(),
+            Self::ReadDocuments(_) => "Read documents".to_string(),
+            Self::EditDocuments(_) => "Edit documents".to_string(),
+            Self::CreateDocuments(_) => "Create documents".to_string(),
+            Self::ReadShellCommandOutput { .. } => "Read shell command output".to_string(),
+            Self::UseComputer(_) => "Use computer".to_string(),
+            Self::InsertCodeReviewComments { comments, .. } => {
+                format!("Insert {} code review comments", comments.len())
+            }
+            Self::RequestComputerUse(_) => "Request computer use".to_string(),
+            Self::ReadSkill(_) => "Read skill".to_string(),
+            Self::FetchConversation { .. } => "Fetch conversation".to_string(),
+            Self::StartAgent { name, .. } => format!("Start agent: {name}"),
+            Self::SendMessageToAgent { subject, .. } => format!("Send message: {subject}"),
+            Self::TransferShellCommandControlToUser { .. } => {
+                "Transfer shell command control to user".to_string()
+            }
+            Self::AskUserQuestion { questions } => {
+                format!("Ask user {} question(s)", questions.len())
+            }
+        }
+    }
+}
+
+impl Display for AIAgentActionType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            AIAgentActionType::RequestCommandOutput {
+                command,
+                is_read_only,
+                uses_pager,
+                ..
+            } => {
+                write!(
+                    f,
+                    "RequestCommandOutput: {command} (read_only: {is_read_only:?}, pager: {uses_pager:?})"
+                )
+            }
+            AIAgentActionType::WriteToLongRunningShellCommand {
+                block_id,
+                input,
+                mode,
+            } => {
+                write!(
+                    f,
+                    "WriteToLongRunningShellCommand (block id: {block_id}): {input:?}, {mode:?}",
+                )
+            }
+            AIAgentActionType::ReadFiles(request) => {
+                write!(f, "{request}")
+            }
+            AIAgentActionType::UploadArtifact(request) => {
+                write!(f, "{request}")
+            }
+            AIAgentActionType::SearchCodebase(request) => {
+                write!(f, "{request}")
+            }
+            AIAgentActionType::RequestFileEdits { file_edits, title } => {
+                let file_names = file_edits
+                    .iter()
+                    .filter_map(|edit| edit.file())
+                    .collect::<Vec<_>>()
+                    .join(", ");
+                if let Some(title) = title {
+                    write!(f, "RequestFileEdits '{title}': [{file_names}]")
+                } else {
+                    write!(f, "RequestFileEdits: [{file_names}]")
+                }
+            }
+            AIAgentActionType::Grep { queries, path } => {
+                write!(f, "Grep: [{}] in {}", queries.join(", "), path)
+            }
+            AIAgentActionType::FileGlob { patterns, path } => {
+                let path_str = path.as_deref().unwrap_or(".");
+                write!(f, "FileGlob: [{}] in {}", patterns.join(", "), path_str)
+            }
+            AIAgentActionType::FileGlobV2 {
+                patterns,
+                search_dir,
+            } => {
+                let path_str = search_dir.as_deref().unwrap_or(".");
+                write!(f, "FileGlobV2: [{}] in {}", patterns.join(", "), path_str)
+            }
+            AIAgentActionType::ReadMCPResource {
+                server_id: _,
+                name,
+                uri,
+            } => {
+                if let Some(uri) = uri {
+                    write!(f, "ReadMCPResource: {name} ({uri})")
+                } else {
+                    write!(f, "ReadMCPResource: {name}")
+                }
+            }
+            AIAgentActionType::CallMCPTool {
+                server_id: _,
+                name,
+                input,
+            } => {
+                write!(f, "CallMCPTool: {name} with input {input:?}")
+            }
+            AIAgentActionType::SuggestNewConversation { message_id } => {
+                write!(f, "SuggestNewConversation: {message_id}")
+            }
+            AIAgentActionType::SuggestPrompt(request) => {
+                write!(f, "SuggestPrompt: {request:?}")
+            }
+            AIAgentActionType::InitProject => {
+                write!(f, "InitProject")
+            }
+            AIAgentActionType::OpenCodeReview => {
+                write!(f, "OpenCodeReview")
+            }
+            AIAgentActionType::ReadDocuments(request) => {
+                let ids: Vec<String> = request
+                    .document_ids
+                    .iter()
+                    .map(|id| id.to_string())
+                    .collect();
+                write!(f, "ReadDocuments: [{}]", ids.join(", "))
+            }
+            AIAgentActionType::EditDocuments(request) => {
+                write!(f, "EditDocuments: {} diffs", request.diffs.len())
+            }
+            AIAgentActionType::CreateDocuments(request) => {
+                write!(f, "CreateDocuments: {} documents", request.documents.len())
+            }
+            AIAgentActionType::ReadShellCommandOutput { delay, block_id } => {
+                let delay = match delay {
+                    Some(ShellCommandDelay::Duration(duration)) => {
+                        format!("{} seconds", duration.as_secs())
+                    }
+                    Some(ShellCommandDelay::OnCompletion) => "on completion".to_string(),
+                    None => "no".to_string(),
+                };
+                write!(
+                    f,
+                    "ReadShellCommandOutput (block id: {block_id}): with {delay} delay"
+                )
+            }
+            AIAgentActionType::UseComputer(req) => {
+                write!(
+                    f,
+                    "UseComputer: {} actions, screenshot_params={:?}",
+                    req.actions.len(),
+                    req.screenshot_params
+                )
+            }
+            AIAgentActionType::InsertCodeReviewComments { comments, .. } => {
+                let file_paths = comments
+                    .iter()
+                    .filter_map(|c| {
+                        c.comment_location
+                            .as_ref()
+                            .map(|loc| loc.relative_file_path.as_str())
+                    })
+                    .collect::<Vec<_>>()
+                    .join(", ");
+                write!(
+                    f,
+                    "InsertCodeReviewComments: {} comments on [{}]",
+                    comments.len(),
+                    file_paths
+                )
+            }
+            AIAgentActionType::RequestComputerUse(req) => {
+                write!(f, "RequestComputerUse: {}", req.task_summary)
+            }
+            AIAgentActionType::ReadSkill(req) => {
+                write!(f, "ReadSkill: {}", req.skill)
+            }
+            AIAgentActionType::FetchConversation { conversation_id } => {
+                write!(f, "FetchConversation: {conversation_id}")
+            }
+            AIAgentActionType::StartAgent { name, .. } => {
+                write!(f, "StartAgent: {name}")
+            }
+            AIAgentActionType::SendMessageToAgent {
+                addresses, subject, ..
+            } => {
+                write!(
+                    f,
+                    "SendMessageToAgent: to=[{}] subject={subject}",
+                    addresses.join(", ")
+                )
+            }
+            AIAgentActionType::TransferShellCommandControlToUser { reason } => {
+                write!(f, "TransferShellCommandControlToUser: {reason}")
+            }
+            AIAgentActionType::AskUserQuestion { questions } => {
+                write!(f, "AskUserQuestion: {} question(s)", questions.len())
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub enum AskUserQuestionType {
+    MultipleChoice {
+        is_multiselect: bool,
+        options: Vec<AskUserQuestionOption>,
+        supports_other: bool,
+    },
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct AskUserQuestionOption {
+    pub label: String,
+    pub recommended: bool,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct AskUserQuestionItem {
+    pub question_id: String,
+    pub question: String,
+    pub question_type: AskUserQuestionType,
+}
+
+impl AskUserQuestionItem {
+    pub fn is_multiselect(&self) -> bool {
+        match &self.question_type {
+            AskUserQuestionType::MultipleChoice { is_multiselect, .. } => *is_multiselect,
+        }
+    }
+
+    pub fn multiple_choice_options(&self) -> Option<&[AskUserQuestionOption]> {
+        match &self.question_type {
+            AskUserQuestionType::MultipleChoice { options, .. } => Some(options),
+        }
+    }
+
+    pub fn supports_other(&self) -> bool {
+        match &self.question_type {
+            AskUserQuestionType::MultipleChoice { supports_other, .. } => *supports_other,
+        }
+    }
+
+    pub fn numbered_option_count(&self) -> usize {
+        self.multiple_choice_options()
+            .map_or(0, |options| options.len())
+            + usize::from(self.supports_other())
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ReadFilesRequest {
+    pub locations: Vec<FileLocations>,
+}
+
+impl Display for ReadFilesRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let file_names = self
+            .locations
+            .iter()
+            .map(|loc| loc.name.as_str())
+            .collect::<Vec<_>>()
+            .join(", ");
+        write!(f, "ReadFiles: [{file_names}]")
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct UploadArtifactRequest {
+    pub file_path: String,
+    pub description: Option<String>,
+}
+
+impl Display for UploadArtifactRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "UploadArtifact: {}", self.file_path)
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct SearchCodebaseRequest {
+    pub query: String,
+
+    /// Optional list of file paths to search through.  This is used to narrow down the search scope.
+    /// Files are searched if any of the partial paths are a substring of the file path.
+    pub partial_paths: Option<Vec<String>>,
+
+    /// Optional absolute path to the codebase that we want to search. If not
+    /// provided, we will use the codebase in the user's current directory.
+    pub codebase_path: Option<String>,
+}
+
+impl Display for SearchCodebaseRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "SearchCodebase: {}", self.query)
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ReadDocumentsRequest {
+    pub document_ids: Vec<AIDocumentId>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct DocumentDiff {
+    pub document_id: AIDocumentId,
+    pub search: String,
+    pub replace: String,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct EditDocumentsRequest {
+    pub diffs: Vec<DocumentDiff>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct DocumentToCreate {
+    pub content: String,
+    pub title: String,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct CreateDocumentsRequest {
+    pub documents: Vec<DocumentToCreate>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct UseComputerRequest {
+    pub action_summary: String,
+    pub actions: Vec<computer_use::Action>,
+    /// If set, a screenshot will be captured after the actions are executed.
+    pub screenshot_params: Option<computer_use::ScreenshotParams>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct RequestComputerUseRequest {
+    /// A short summary of the task.
+    pub task_summary: String,
+    /// If set, a screenshot will be captured after the actions are executed.
+    pub screenshot_params: Option<computer_use::ScreenshotParams>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ReadSkillRequest {
+    pub skill: SkillReference,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ShellCommandDelay {
+    Duration(Duration),
+    OnCompletion,
+}
+
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, EnumDiscriminants)]
+pub enum AIAgentPtyWriteMode {
+    #[default]
+    Raw,
+    Line,
+    Block,
+}
+
+impl AIAgentPtyWriteMode {
+    /// Decorates input bytes according to the write mode.
+    pub fn decorate_bytes(
+        self,
+        bytes: impl Into<Vec<u8>>,
+        is_bracketed_paste_enabled: bool,
+    ) -> Vec<u8> {
+        use warp_terminal::model::escape_sequences;
+
+        let bytes = bytes.into();
+        match self {
+            AIAgentPtyWriteMode::Raw => bytes,
+            AIAgentPtyWriteMode::Line => {
+                // Move to beginning of line, write input, then submit (Enter).
+                let mut v = Vec::with_capacity(bytes.len() + 2);
+                // ^A (SOH) is "beginning of line" for readline/prompt-toolkit style editors.
+                v.push(escape_sequences::C0::SOH);
+                v.extend_from_slice(&bytes);
+                cfg_if::cfg_if! {
+                    if #[cfg(target_os = "windows")] {
+                        // Use CR to submit on Windows hosts.
+                        v.push(escape_sequences::C0::CR);
+                    } else {
+                        // Use LF to submit on POSIX.
+                        v.push(escape_sequences::C0::LF);
+                    }
+                }
+                v
+            }
+            AIAgentPtyWriteMode::Block => {
+                if is_bracketed_paste_enabled {
+                    escape_sequences::BRACKETED_PASTE_START
+                        .iter()
+                        .copied()
+                        .chain(bytes)
+                        .chain(escape_sequences::BRACKETED_PASTE_END.iter().copied())
+                        .collect()
+                } else {
+                    bytes
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct InsertReviewComment {
+    pub comment_id: String,
+    pub author: String,
+    pub last_modified_timestamp: String,
+    pub comment_body: String,
+    pub parent_comment_id: Option<String>,
+    /// The file and line range the comment is attached to.
+    /// If None, the comment applies to the whole diff set.
+    pub comment_location: Option<InsertedCommentLocation>,
+    pub html_url: Option<String>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct InsertedCommentLocation {
+    /// Repo-relative path of the file the comment is attached to.
+    pub relative_file_path: String,
+    /// The specific line range the comment is attached to.
+    /// If None, the comment applies to the whole file.
+    pub line: Option<InsertedCommentLine>,
+}
+
+/// The side of a diff that a comment is attached to.
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum CommentSide {
+    /// The right side of the diff (new file / additions).
+    Right,
+    /// The left side of the diff (old file / deletions).
+    Left,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct InsertedCommentLine {
+    pub comment_line_range: Range<usize>,
+    /// The diff hunk line range overlaps with the comment line range
+    /// but may not match it exactly. We need this in order to be able
+    /// to find the full diff hunk this comment is attached to.
+    pub diff_hunk_line_range: Range<usize>,
+    /// The diff hunk text is needed to find where to attach comments
+    /// when line numbers on the local and remote branches have diverged.
+    pub diff_hunk_text: String,
+    /// The side of the diff the comment is attached to.
+    pub side: Option<CommentSide>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum SuggestPromptRequest {
+    UnitTestsSuggestion {
+        query: String,
+        title: String,
+        description: String,
+    },
+    PromptSuggestion {
+        prompt: String,
+        label: Option<String>,
+    },
+}
+
+/// A file-editing request from the agent.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum FileEdit {
+    /// Edit an existing file by applying a diff.
+    Edit(ParsedDiff),
+    /// Create a new file.
+    Create {
+        file: Option<String>,
+        content: Option<String>,
+    },
+    /// Delete an existing file.
+    Delete { file: Option<String> },
+}
+
+impl FileEdit {
+    /// The path to the file this edit applies to.
+    pub fn file(&self) -> Option<&str> {
+        match self {
+            Self::Edit(diff) => diff.file().map(|s| s.as_str()),
+            Self::Create { file, .. } => file.as_deref(),
+            Self::Delete { file } => file.as_deref(),
+        }
+    }
+}

crates/ai/src/agent/action_result/convert_tests.rs

@@ -0,0 +1,30 @@
+use super::*;
+
+#[test]
+fn ask_user_question_skipped_by_auto_approve_converts_to_skipped_answers() {
+    let result = api::request::input::tool_call_result::Result::from(
+        AskUserQuestionResult::SkippedByAutoApprove {
+            question_ids: vec!["q1".to_string(), "q2".to_string()],
+        },
+    );
+
+    let api::request::input::tool_call_result::Result::AskUserQuestion(result) = result else {
+        panic!("expected ask_user_question result");
+    };
+
+    let Some(api::ask_user_question_result::Result::Success(success)) = result.result else {
+        panic!("expected success result");
+    };
+
+    assert_eq!(success.answers.len(), 2);
+    assert_eq!(success.answers[0].question_id, "q1");
+    assert_eq!(success.answers[1].question_id, "q2");
+    assert!(matches!(
+        success.answers[0].answer,
+        Some(AskUserQuestionAnswer::Skipped(()))
+    ));
+    assert!(matches!(
+        success.answers[1].answer,
+        Some(AskUserQuestionAnswer::Skipped(()))
+    ));
+}

crates/ai/src/agent/action_result/mod_tests.rs

@@ -0,0 +1,44 @@
+use super::{StartAgentResult, StartAgentVersion};
+
+#[test]
+fn deserializes_legacy_start_agent_success_without_version_as_v1() {
+    let result: StartAgentResult =
+        serde_json::from_value(serde_json::json!({ "Success": { "agent_id": "agent-1" } }))
+            .expect("legacy start-agent success should deserialize");
+
+    assert_eq!(
+        result,
+        StartAgentResult::Success {
+            agent_id: "agent-1".to_string(),
+            version: StartAgentVersion::V1,
+        }
+    );
+}
+
+#[test]
+fn deserializes_legacy_start_agent_error_without_version_as_v1() {
+    let result: StartAgentResult =
+        serde_json::from_value(serde_json::json!({ "Error": { "error": "boom" } }))
+            .expect("legacy start-agent error should deserialize");
+
+    assert_eq!(
+        result,
+        StartAgentResult::Error {
+            error: "boom".to_string(),
+            version: StartAgentVersion::V1,
+        }
+    );
+}
+
+#[test]
+fn deserializes_legacy_start_agent_cancelled_without_version_as_v1() {
+    let result: StartAgentResult = serde_json::from_value(serde_json::json!({ "Cancelled": {} }))
+        .expect("legacy start-agent cancellation should deserialize");
+
+    assert_eq!(
+        result,
+        StartAgentResult::Cancelled {
+            version: StartAgentVersion::V1,
+        }
+    );
+}

crates/ai/src/agent/citation.rs

@@ -0,0 +1,57 @@
+use std::fmt::Display;
+
+use warp_multi_agent_api as api;
+
+/// A citation listed in an AI response.
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub enum AIAgentCitation {
+    WarpDriveObject { uid: String },
+    WarpDocumentation { path: String },
+    WebPage { url: String },
+}
+
+impl Display for AIAgentCitation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            AIAgentCitation::WarpDriveObject { uid } => {
+                write!(f, "Warp Drive Object: {uid}")
+            }
+            AIAgentCitation::WarpDocumentation { path } => {
+                write!(f, "Warp Documentation: {path}")
+            }
+            AIAgentCitation::WebPage { url } => {
+                write!(f, "Web Page: {url}")
+            }
+        }
+    }
+}
+
+/// Error type for Citation conversion errors
+#[derive(Debug, thiserror::Error)]
+#[error("Unknown citation type")]
+pub struct UnknownCitationTypeError;
+
+impl TryFrom<api::Citation> for AIAgentCitation {
+    type Error = UnknownCitationTypeError;
+
+    fn try_from(citation: api::Citation) -> Result<Self, Self::Error> {
+        let doc_type = api::DocumentType::try_from(citation.document_type)
+            .unwrap_or(api::DocumentType::Unknown);
+
+        match doc_type {
+            api::DocumentType::WarpDriveWorkflow
+            | api::DocumentType::WarpDriveNotebook
+            | api::DocumentType::WarpDriveEnvVar
+            | api::DocumentType::Rule => Ok(AIAgentCitation::WarpDriveObject {
+                uid: citation.document_id,
+            }),
+            api::DocumentType::WarpDocumentation => Ok(AIAgentCitation::WarpDocumentation {
+                path: citation.document_id,
+            }),
+            api::DocumentType::WebPage => Ok(AIAgentCitation::WebPage {
+                url: citation.document_id,
+            }),
+            api::DocumentType::Unknown => Err(UnknownCitationTypeError),
+        }
+    }
+}

crates/ai/src/agent/convert.rs

@@ -0,0 +1,41 @@
+#[derive(thiserror::Error, Debug)]
+pub enum ConvertToAPITypeError {
+    /// There is no API type for the given value.
+    ///
+    /// This means the value just be ignored during request construction.
+    #[error("Ignoring value when constructing API type.")]
+    Ignore,
+    #[error("Conversion from type {0} is unimplemented.")]
+    Unimplemented(String),
+    #[error("Encountered error converting types for MultiAgentApi request: {0:?}")]
+    Other(#[from] anyhow::Error),
+}
+
+/// Unexpected errors when trying to convert an [`api::message::ToolCall`] to an [`AIAgentAction`].
+#[derive(Debug, thiserror::Error)]
+pub enum ToolToAIAgentActionError {
+    #[error("Missing tool")]
+    MissingTool,
+    #[error("Could not parse args for MCP tool call: {0}")]
+    CallMCPToolArgsError(String),
+    #[error("Error converting suggest prompt tool call: {0}")]
+    SuggestPromptError(String),
+    #[error("Required coordinates for computer use action were missing")]
+    MissingComputerUseCoordinates,
+    #[error("Required scroll distance for mouse wheel action was missing")]
+    MissingComputerUseScrollDistance,
+    #[error("Received missing computer use action type")]
+    MissingComputerUseActionType,
+    #[error("Wait duration must be non-negative")]
+    InvalidComputerUseWaitDuration,
+    #[error("Required key for KeyDown/KeyUp action was missing")]
+    MissingComputerUseKey,
+    #[error("Character key was empty")]
+    InvalidComputerUseCharKey,
+    #[error("Received unexpected tool")]
+    UnexpectedTool,
+    #[error("Missing required reference for read skill tool call")]
+    MissingSkillReference,
+    #[error("Missing required file reference for upload artifact tool call")]
+    MissingUploadArtifactFileReference,
+}

crates/ai/src/agent/file_locations.rs

@@ -0,0 +1,146 @@
+use std::collections::HashMap;
+use std::ops::Range;
+
+use itertools::Itertools as _;
+use warp_terminal::shell::ShellLaunchData;
+
+use crate::agent::action_result::FileContext;
+use crate::{index::locations::CodeContextLocation, paths::shell_native_absolute_path};
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct FileLocations {
+    pub name: String,
+    pub lines: Vec<Range<usize>>,
+}
+
+impl FileLocations {
+    /// Convert file locations to a user readable format.
+    pub fn to_user_message(
+        &self,
+        shell_launch_data: Option<&ShellLaunchData>,
+        current_working_directory: Option<&String>,
+        file_line_count: Option<usize>,
+    ) -> String {
+        let absolute_path =
+            shell_native_absolute_path(&self.name, shell_launch_data, current_working_directory);
+
+        if self.lines.is_empty() {
+            return absolute_path;
+        }
+
+        let line_ranges = self
+            .lines
+            .iter()
+            .filter_map(|range| {
+                let (start, end) = match file_line_count {
+                    Some(line_count) => (
+                        std::cmp::min(range.start, line_count),
+                        std::cmp::min(range.end, line_count),
+                    ),
+                    None => (range.start, range.end),
+                };
+
+                if start == 1 && Some(end) == file_line_count {
+                    // don't show ranges that are just the entire file
+                    None
+                } else {
+                    Some(format!("{start}-{end}"))
+                }
+            })
+            .collect_vec();
+
+        if line_ranges.is_empty() {
+            absolute_path
+        } else {
+            format!("{} ({})", absolute_path, line_ranges.join(", "))
+        }
+    }
+
+    /// Expands the line ranges (if any) in both directions by `context_line`. Then the line ranges are sorted
+    /// and merged if there are overlaps.
+    pub fn expand_surrounding_context(&mut self, context_line: usize) {
+        if self.lines.is_empty() {
+            return;
+        }
+
+        // Expand each range by context_line in both directions
+        let mut expanded: Vec<Range<usize>> = self
+            .lines
+            .iter()
+            .map(|r| {
+                let start = r.start.saturating_sub(context_line);
+                let end = r.end + context_line;
+                start..end
+            })
+            .collect();
+
+        // Sort by start
+        expanded.sort_by_key(|r| r.start);
+
+        // Merge overlapping or adjacent ranges
+        let mut merged: Vec<Range<usize>> = Vec::with_capacity(expanded.len());
+        for range in expanded {
+            if let Some(last) = merged.last_mut() {
+                if range.start <= last.end {
+                    last.end = last.end.max(range.end);
+                } else {
+                    merged.push(range);
+                }
+            } else {
+                merged.push(range);
+            }
+        }
+        self.lines = merged;
+    }
+}
+
+impl From<&CodeContextLocation> for FileLocations {
+    fn from(location: &CodeContextLocation) -> Self {
+        match location {
+            CodeContextLocation::WholeFile(path) => Self {
+                name: path.to_string_lossy().to_string(),
+                lines: vec![],
+            },
+            CodeContextLocation::Fragment(fragment) => Self {
+                name: fragment.path.to_string_lossy().to_string(),
+                lines: fragment.line_ranges.clone(),
+            },
+        }
+    }
+}
+
+/// Groups a slice of [`FileContext`]s by file name, collecting line ranges from
+/// fragments of the same file. Returns one display string per unique file,
+/// preserving first-occurrence order.
+pub fn group_file_contexts_for_display(
+    file_contexts: &[FileContext],
+    shell_launch_data: Option<&ShellLaunchData>,
+    current_working_directory: Option<&String>,
+) -> Vec<String> {
+    let mut order: Vec<String> = Vec::new();
+    let mut groups: HashMap<String, Vec<Range<usize>>> = HashMap::new();
+
+    for fc in file_contexts {
+        let entry = groups.entry(fc.file_name.clone()).or_insert_with(|| {
+            order.push(fc.file_name.clone());
+            Vec::new()
+        });
+        if let Some(range) = &fc.line_range {
+            entry.push(range.clone());
+        }
+    }
+
+    order
+        .iter()
+        .map(|file_name| {
+            let ranges = groups.get(file_name).unwrap();
+            let mut sorted = ranges.clone();
+            sorted.sort_by_key(|r| (r.start, r.end));
+            let locations = FileLocations {
+                name: file_name.clone(),
+                lines: sorted,
+            };
+            locations.to_user_message(shell_launch_data, current_working_directory, None)
+        })
+        .collect()
+}

crates/ai/src/agent/mod.rs

@@ -0,0 +1,8 @@
+pub mod action;
+pub mod action_result;
+mod citation;
+pub mod convert;
+pub mod file_locations;
+
+pub use citation::{AIAgentCitation, UnknownCitationTypeError};
+pub use file_locations::{group_file_contexts_for_display, FileLocations};

crates/ai/src/api_keys.rs

@@ -0,0 +1,219 @@
+pub use crate::aws_credentials::{AwsCredentials, AwsCredentialsState};
+use serde::{Deserialize, Serialize};
+use warp_multi_agent_api as api;
+use warpui::{Entity, ModelContext, SingletonEntity};
+use warpui_extras::secure_storage::{self, AppContextExt};
+
+const SECURE_STORAGE_KEY: &str = "AiApiKeys";
+
+/// Emitted when user-provided API keys are updated in-memory.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ApiKeyManagerEvent {
+    KeysUpdated,
+}
+
+/// User-provided API keys for AI providers.
+///
+/// These are used for "Bring Your Own API Key" functionality, allowing
+/// users to use their own API keys instead of Warp's.
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+pub struct ApiKeys {
+    pub google: Option<String>,
+    pub anthropic: Option<String>,
+    pub openai: Option<String>,
+    pub open_router: Option<String>,
+}
+
+impl ApiKeys {
+    pub fn has_any_key(&self) -> bool {
+        self.openai.is_some()
+            || self.anthropic.is_some()
+            || self.google.is_some()
+            || self.open_router.is_some()
+    }
+}
+
+/// Controls how AWS credentials are refreshed by [`ApiKeyManager`].
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub enum AwsCredentialsRefreshStrategy {
+    /// Load credentials from the local AWS credential chain (~/.aws). This is the default.
+    #[default]
+    LocalChain,
+    /// Credentials are managed externally via OIDC/STS.
+    /// The task ID is used to scope the STS AssumeRoleWithWebIdentity session.
+    /// The role ARN is the IAM role to assume via STS.
+    OidcManaged {
+        task_id: Option<String>,
+        role_arn: String,
+    },
+}
+
+/// A structure that manages API keys for AI providers.
+pub struct ApiKeyManager {
+    keys: ApiKeys,
+    pub(crate) aws_credentials_state: AwsCredentialsState,
+    aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy,
+}
+
+impl ApiKeyManager {
+    pub fn new(ctx: &mut ModelContext<Self>) -> Self {
+        let keys = Self::load_keys_from_secure_storage(ctx);
+        Self {
+            keys,
+            aws_credentials_state: AwsCredentialsState::Missing,
+            aws_credentials_refresh_strategy: AwsCredentialsRefreshStrategy::default(),
+        }
+    }
+
+    pub fn keys(&self) -> &ApiKeys {
+        &self.keys
+    }
+
+    pub fn set_google_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
+        self.keys.google = key;
+        ctx.emit(ApiKeyManagerEvent::KeysUpdated);
+        self.write_keys_to_secure_storage(ctx);
+    }
+
+    pub fn set_anthropic_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
+        self.keys.anthropic = key;
+        ctx.emit(ApiKeyManagerEvent::KeysUpdated);
+        self.write_keys_to_secure_storage(ctx);
+    }
+
+    pub fn set_openai_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
+        self.keys.openai = key;
+        ctx.emit(ApiKeyManagerEvent::KeysUpdated);
+        self.write_keys_to_secure_storage(ctx);
+    }
+
+    pub fn set_open_router_key(&mut self, key: Option<String>, ctx: &mut ModelContext<Self>) {
+        self.keys.open_router = key;
+        ctx.emit(ApiKeyManagerEvent::KeysUpdated);
+        self.write_keys_to_secure_storage(ctx);
+    }
+
+    pub fn set_aws_credentials_state(
+        &mut self,
+        state: AwsCredentialsState,
+        ctx: &mut ModelContext<Self>,
+    ) {
+        self.aws_credentials_state = state;
+        ctx.emit(ApiKeyManagerEvent::KeysUpdated);
+    }
+
+    pub fn aws_credentials_state(&self) -> &AwsCredentialsState {
+        &self.aws_credentials_state
+    }
+
+    pub fn aws_credentials_refresh_strategy(&self) -> AwsCredentialsRefreshStrategy {
+        self.aws_credentials_refresh_strategy.clone()
+    }
+
+    pub fn set_aws_credentials_refresh_strategy(
+        &mut self,
+        strategy: AwsCredentialsRefreshStrategy,
+    ) {
+        self.aws_credentials_refresh_strategy = strategy;
+    }
+
+    pub fn api_keys_for_request(
+        &self,
+        include_byo_keys: bool,
+        include_aws_bedrock_credentials: bool,
+    ) -> Option<api::request::settings::ApiKeys> {
+        let anthropic = include_byo_keys
+            .then(|| self.keys.anthropic.clone())
+            .flatten()
+            .unwrap_or_default();
+        let openai = include_byo_keys
+            .then(|| self.keys.openai.clone())
+            .flatten()
+            .unwrap_or_default();
+        let google = include_byo_keys
+            .then(|| self.keys.google.clone())
+            .flatten()
+            .unwrap_or_default();
+        let open_router = include_byo_keys
+            .then(|| self.keys.open_router.clone())
+            .flatten()
+            .unwrap_or_default();
+        // Also include credentials when running with OIDC-managed Bedrock inference, regardless
+        // of the per-user setting flag (which only applies to the local credential chain path).
+        let include_aws = include_aws_bedrock_credentials
+            || matches!(
+                self.aws_credentials_refresh_strategy,
+                AwsCredentialsRefreshStrategy::OidcManaged { .. }
+            );
+        let aws_credentials = include_aws
+            .then(|| match self.aws_credentials_state {
+                AwsCredentialsState::Loaded {
+                    ref credentials, ..
+                } => Some(credentials.clone().into()),
+                _ => None,
+            })
+            .flatten();
+
+        if anthropic.is_empty()
+            && openai.is_empty()
+            && google.is_empty()
+            && open_router.is_empty()
+            && aws_credentials.is_none()
+        {
+            None
+        } else {
+            Some(api::request::settings::ApiKeys {
+                anthropic,
+                openai,
+                google,
+                open_router,
+                allow_use_of_warp_credits: false,
+                aws_credentials,
+            })
+        }
+    }
+
+    fn load_keys_from_secure_storage(ctx: &mut ModelContext<Self>) -> ApiKeys {
+        let key_json = match ctx.secure_storage().read_value(SECURE_STORAGE_KEY) {
+            Ok(json) => json,
+            Err(e) => {
+                if !matches!(e, secure_storage::Error::NotFound) {
+                    log::error!("Failed to read API keys from secure storage: {e:#}");
+                }
+                return ApiKeys::default();
+            }
+        };
+
+        let keys = match serde_json::from_str(&key_json) {
+            Ok(keys) => keys,
+            Err(e) => {
+                log::error!("Failed to deserialize API keys: {e:#}");
+                ApiKeys::default()
+            }
+        };
+
+        keys
+    }
+
+    fn write_keys_to_secure_storage(&mut self, ctx: &mut ModelContext<Self>) {
+        let keys = self.keys.clone();
+
+        let json = match serde_json::to_string(&keys) {
+            Ok(json) => json,
+            Err(e) => {
+                log::error!("Failed to serialize API keys: {e:#}");
+                return;
+            }
+        };
+
+        if let Err(e) = ctx.secure_storage().write_value(SECURE_STORAGE_KEY, &json) {
+            log::error!("Failed to write API keys to secure storage: {e:#}");
+        }
+    }
+}
+
+impl Entity for ApiKeyManager {
+    type Event = ApiKeyManagerEvent;
+}
+
+impl SingletonEntity for ApiKeyManager {}

crates/ai/src/aws_credentials.rs

@@ -0,0 +1,113 @@
+use std::time::SystemTime;
+
+use chrono::{DateTime, Local};
+use warp_core::ui::Icon;
+use warp_multi_agent_api as api;
+
+/// Temporary AWS credentials loaded from the AWS SDK.
+/// These are not persisted and are only used at runtime.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AwsCredentials {
+    access_key: String,
+    secret_key: String,
+    session_token: Option<String>,
+    expires_at: Option<SystemTime>,
+}
+
+impl AwsCredentials {
+    pub fn new(
+        access_key: String,
+        secret_key: String,
+        session_token: Option<String>,
+        expires_at: Option<SystemTime>,
+    ) -> Self {
+        Self {
+            access_key,
+            secret_key,
+            session_token,
+            expires_at,
+        }
+    }
+
+    pub fn expires_at(&self) -> Option<SystemTime> {
+        self.expires_at
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum AwsCredentialsState {
+    Missing,
+    Disabled,
+    Refreshing,
+    Loaded {
+        credentials: AwsCredentials,
+        loaded_at: SystemTime,
+    },
+    Failed {
+        message: String,
+    },
+}
+
+impl From<AwsCredentials> for api::request::settings::api_keys::AwsCredentials {
+    fn from(creds: AwsCredentials) -> Self {
+        Self {
+            access_key: creds.access_key,
+            secret_key: creds.secret_key,
+            session_token: creds.session_token.unwrap_or_default(),
+            region: String::new(),
+        }
+    }
+}
+
+fn format_status_timestamp(time: SystemTime) -> String {
+    let datetime: DateTime<Local> = time.into();
+    if datetime.date_naive() == Local::now().date_naive() {
+        datetime.format("%-I:%M %p").to_string()
+    } else {
+        datetime.format("%b %-d at %-I:%M %p").to_string()
+    }
+}
+
+impl AwsCredentialsState {
+    pub fn user_facing_components(&self) -> (String, String, Icon) {
+        match self {
+            Self::Missing => (
+                "AWS credentials not configured".to_string(),
+                "Log in to the AWS CLI or configure AWS credentials for this profile, then refresh."
+                    .to_string(),
+                Icon::Key,
+            ),
+            Self::Disabled => (
+                "AWS Bedrock Disabled".to_string(),
+                "Warp will not load your AWS CLI credentials until AWS Bedrock is enabled by you or your workspace admin"
+                    .to_string(),
+                Icon::Key,
+            ),
+            Self::Refreshing => (
+                "Refreshing credentials...".to_string(),
+                "Loading your AWS CLI credentials into Warp".to_string(),
+                Icon::RefreshCw04,
+            ),
+            Self::Loaded {
+                credentials,
+                loaded_at,
+            } => (
+                "Credentials loaded".to_string(),
+                match credentials.expires_at() {
+                    Some(expires_at) => format!(
+                        "Loaded at {}, expires {}",
+                        format_status_timestamp(*loaded_at),
+                        format_status_timestamp(expires_at)
+                    ),
+                    None => format!("Loaded at {}", format_status_timestamp(*loaded_at)),
+                },
+                Icon::CheckCircleBroken,
+            ),
+            Self::Failed { message } => (
+                "Unable to load credentials".to_string(),
+                message.clone(),
+                Icon::AlertTriangle,
+            ),
+        }
+    }
+}

crates/ai/src/diff_validation/mod_test.rs

@@ -0,0 +1,681 @@
+use std::vec;
+
+use super::*;
+
+fn deltas(diff: &AIRequestedCodeDiff) -> &[DiffDelta] {
+    match &diff.diff_type {
+        DiffType::Update { deltas, .. } => deltas,
+        other => panic!("Expected Update diff_type, got {other:?}"),
+    }
+}
+
+const CONTENT: &str = "I'd just like to interject
+                        for a moment. What you're refering to as
+                        Linux, is in fact, GNU/Linux, or as I've
+                        recently taken to calling it, GNU plus
+                        Linux. Linux is not an operating system
+                        unto itself, but rather another free
+                        component of a fully functioning GNU
+                        system made useful by the GNU corelibs,
+                        shell utilities and vital system
+                        components comprising a full OS as
+                        defined by POSIX.";
+
+#[test]
+fn test_simple() {
+    let input_diffs = vec![
+        SearchAndReplace {
+            search: "2|hey".to_string(),
+            replace: "what".to_string(),
+        },
+        SearchAndReplace {
+            search: "4|world\n5|of".to_string(),
+            replace: "hey".to_string(),
+        },
+    ];
+
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, "what\nhey\nthere\nworld\nof\n");
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(
+        deltas(&diff),
+        &[
+            DiffDelta {
+                replacement_line_range: 2..3,
+                insertion: "what".to_string(),
+            },
+            DiffDelta {
+                replacement_line_range: 4..6,
+                insertion: "hey".to_string(),
+            }
+        ]
+    );
+}
+
+#[test]
+fn test_incorrect_line_numbers() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "4|world\n5|of".to_string(),
+        replace: "hey".to_string(),
+    }];
+
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, "what\nthere\nworld\nof");
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 3..5,
+            insertion: "hey".to_string(),
+        }]
+    );
+}
+
+#[test]
+fn test_missing_line_numbers() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "hey\nthere".to_string(),
+        replace: "world".to_string(),
+    }];
+
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, "what\nhey\nthere\nworld\nof\n");
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 2..4,
+            insertion: "world".to_string(),
+        }]
+    );
+
+    let failures = diff.failures.expect("Expected failures to be tracked");
+    assert_eq!(failures.missing_line_numbers, 1);
+    assert_eq!(failures.fuzzy_match_failures, 0);
+    assert_eq!(failures.noop_deltas, 0);
+}
+
+#[test]
+fn test_blank_search() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "".to_string(),
+        replace: "hey".to_string(),
+    }];
+
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, "what\nhey\nthere\nworld\nof\n");
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 0..0,
+            insertion: "hey".to_string(),
+        }]
+    );
+}
+
+#[test]
+fn test_closest() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "4|world\n5|of".to_string(),
+        replace: "hey".to_string(),
+    }];
+
+    let diff = fuzzy_match_diffs(
+        "test.rs",
+        &input_diffs,
+        "what\nhey\nworld\nof\nthe\nworld\nof\n",
+    );
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 3..5,
+            insertion: "hey".to_string(),
+        }]
+    );
+}
+
+#[test]
+fn test_line_numbers_off_by_one() {
+    let insertion = "                        Linux, is in fact, GNU/Linux, or as I've
+                        recently taken to calling it, GNU plus
+                        Linux. Linux is not an operating system
+                        unto itself, but rather another free
+                        component of a fully functioning GNU
+                        system made useful by the GNU corelibs,
+                        hello, world!"
+        .to_string();
+    let input_diffs = vec![SearchAndReplace {
+        search: "2|                        Linux, is in fact, GNU/Linux, or as I've\n\
+                 3|                        recently taken to calling it, GNU plus\n\
+                 4|                        Linux. Linux is not an operating system\n\
+                 5|                        unto itself, but rather another free\n\
+                 6|                        component of a fully functioning GNU\n\
+                 7|                        system made useful by the GNU corelibs,"
+            .to_string(),
+        replace: insertion.clone(),
+    }];
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, CONTENT);
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 3..9,
+            insertion,
+        }]
+    );
+}
+
+#[test]
+fn test_append_to_end_of_file() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "3|".to_string(),
+        replace: "foo".to_string(),
+    }];
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, "\n\n\n");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 3..4,
+            insertion: "foo".to_string(),
+        }]
+    )
+}
+
+#[test]
+fn test_totally_unrelated_search() {
+    let input_diffs = vec![SearchAndReplace {
+        search: "4|foo bar baz".to_string(),
+        replace: "hello, world!".to_string(),
+    }];
+    let diff = fuzzy_match_diffs("test.rs", &input_diffs, CONTENT);
+    assert!(deltas(&diff).is_empty());
+    assert!(diff.failures.is_some());
+}
+
+/// The agent sometimes emits a search whose final line is a prefix of the actual file line.
+/// Before `PrefixTailMatch`, the Jaro-Winkler scorer landed just under the 0.9 threshold for
+/// long lines and the diff failed with `Could not apply all diffs to <file>`.  With
+/// `PrefixTailMatch` in the cascade, the rescue succeeds and the existing suffix-preservation
+/// fixup splices the unmatched tail into the insertion.
+#[test]
+fn test_prefix_tail_rescue_with_line_number_hint() {
+    let actual_line = "if the stripping tool encounters any error (nesting, unmatched markers, UTF-8 decode failure), the sync workflow **fails** and does **not** update the watermark.  the next run will retry from the same commit.  this is correct fail-closed behavior \u{2014} a stripping error might indicate a condition that could cause private code to leak.";
+    let file_content = format!("(preamble)\n\n### error handling\n\n{actual_line}\n\n(trailer)\n");
+
+    // Search is a prefix of line 5, with the `5|` line-number hint.
+    let search = "5|if the stripping tool encounters any error (nesting, unmatched markers, UTF-8 decode failure), the sync workflow **fails** and does **not** update the watermark.";
+    let replace = "if the stripping tool encounters any error (nesting, unmatched markers, UTF-8 decode failure, symlinks), the sync workflow **fails** and does **not** update the watermark.";
+
+    let input_diffs = vec![SearchAndReplace {
+        search: search.to_string(),
+        replace: replace.to_string(),
+    }];
+
+    let diff = fuzzy_match_diffs("TECH-DESIGN.md", &input_diffs, &file_content);
+
+    // The rescue should produce a single delta replacing line 5 with the replacement
+    // plus the unmatched suffix of the original line appended by the existing fixup.
+    let unmatched_suffix = &actual_line[search.strip_prefix("5|").unwrap().len()..];
+    let expected_insertion = format!("{replace}{unmatched_suffix}");
+    assert_eq!(
+        deltas(&diff),
+        &[DiffDelta {
+            replacement_line_range: 5..6,
+            insertion: expected_insertion,
+        }]
+    );
+
+    // The rescue succeeds cleanly — no failure signals should be surfaced.
+    assert!(diff.failures.is_none());
+    assert!(!diff.warrants_failure());
+}
+
+#[test]
+fn test_parse_line_numbers() {
+    let search = "1|hey\n2|there\n3|world";
+    let (line_range, line) = parse_line_numbers(search);
+    assert_eq!(line_range, Some(1..4));
+    assert_eq!(line, "hey\nthere\nworld");
+
+    let search = "hey\nthere";
+    let (line_range, line) = parse_line_numbers(search);
+    assert_eq!(line_range, None);
+    assert_eq!(line, "hey\nthere");
+
+    let search = "";
+    let (line_range, line) = parse_line_numbers(search);
+    assert_eq!(line_range, Some(0..0));
+    assert_eq!(line, "");
+}
+
+#[test]
+fn test_remove_extra_line_num_prefix() {
+    // Test with line numbers.
+    let input = "1|first line\n2|second line\n3|third line".to_string();
+    assert_eq!(
+        remove_extra_line_num_prefix(input),
+        "first line\nsecond line\nthird line"
+    );
+
+    // Test with no line numbers.
+    let input = "first line\nsecond line".to_string();
+    assert_eq!(
+        remove_extra_line_num_prefix(input),
+        "first line\nsecond line"
+    );
+
+    // Test empty string.
+    assert_eq!(remove_extra_line_num_prefix("".to_string()), "");
+
+    // Test single line with number.
+    assert_eq!(
+        remove_extra_line_num_prefix("1|only line".to_string()),
+        "only line"
+    );
+
+    // Test with line numbers with mixed prefixes.
+    let input = "first line\n2|second line\n3|third line".to_string();
+    assert_eq!(
+        remove_extra_line_num_prefix(input),
+        "first line\nsecond line\nthird line"
+    );
+
+    // Test single line without number.
+    let input = "no number line".to_string();
+    assert_eq!(remove_extra_line_num_prefix(input.clone()), input);
+}
+
+#[test]
+fn test_find_similar_sections_out_of_bounds() {
+    let matches = find_similar_sections("hey\nthere\nyou", &[], 0.9);
+    assert!(matches.is_empty());
+
+    let matches = find_similar_sections("hey\nthere\nyou", &["hey", "there", "you"], 0.9);
+    assert_eq!(
+        matches,
+        vec![Match {
+            start_line: 1,
+            end_line: 4,
+            similarity: 1.0
+        }]
+    );
+
+    let matches = find_similar_sections("hey\nthere\nyou", &["hey", "there"], 0.9);
+    assert!(matches.is_empty());
+
+    let matches = find_similar_sections("", &[], 0.9);
+    assert!(matches.is_empty());
+}
+
+#[test]
+fn test_v4a_exact_match() {
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "fn main() {".to_string(),
+        old: "    println!(\"Hello\");".to_string(),
+        new: "    println!(\"Hello, World!\");".to_string(),
+        post_context: "}".to_string(),
+    }];
+
+    let file_content = "fn main() {\n    println!(\"Hello\");\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(diff.file_name, "test.rs");
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(
+        deltas(&diff)[0],
+        DiffDelta {
+            replacement_line_range: 2..3,
+            insertion: "    println!(\"Hello, World!\");".to_string(),
+        }
+    );
+}
+
+#[test]
+fn test_v4a_with_change_context() {
+    let hunks = vec![V4AHunk {
+        change_context: vec!["impl MyStruct {".to_string()],
+        pre_context: "    fn method1() {\n        // comment".to_string(),
+        old: "        let x = 1;".to_string(),
+        new: "        let x = 2;".to_string(),
+        post_context: "    }\n}".to_string(),
+    }];
+
+    let file_content = "struct MyStruct {}\n\nimpl MyStruct {\n    fn method1() {\n        // comment\n        let x = 1;\n    }\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(
+        deltas(&diff)[0],
+        DiffDelta {
+            replacement_line_range: 6..7,
+            insertion: "        let x = 2;".to_string(),
+        }
+    );
+}
+
+#[test]
+fn test_v4a_indentation_agnostic_match() {
+    // Hunk has different indentation than the actual file
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "def hello():".to_string(),
+        old: "print(\"hello\")".to_string(), // No indentation
+        new: "    print(\"hello world\")".to_string(),
+        post_context: "".to_string(),
+    }];
+
+    let file_content = "def hello():\n    print(\"hello\")"; // Has indentation
+    let diff = fuzzy_match_v4a_diffs("test.py", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(
+        deltas(&diff)[0],
+        DiffDelta {
+            replacement_line_range: 2..3,
+            insertion: "    print(\"hello world\")".to_string(),
+        }
+    );
+}
+
+#[test]
+fn test_v4a_fuzzy_match() {
+    // Hunk has slightly different content (typo)
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "function greet() {".to_string(),
+        old: "    console.log(\"helo\");".to_string(), // Typo: "helo" instead of "hello"
+        new: "    console.log(\"hello world\");".to_string(),
+        post_context: "}".to_string(),
+    }];
+
+    let file_content = "function greet() {\n    console.log(\"hello\");\n}"; // Correct spelling
+    let diff = fuzzy_match_v4a_diffs("test.js", &hunks, None, file_content);
+
+    // Should match due to high similarity (> 0.9)
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(deltas(&diff)[0].replacement_line_range, 2..3);
+}
+
+#[test]
+fn test_v4a_no_match() {
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "fn does_not_exist() {".to_string(),
+        old: "    unrelated_code();".to_string(),
+        new: "    new_code();".to_string(),
+        post_context: "}".to_string(),
+    }];
+
+    let file_content = "fn main() {\n    println!(\"Hello\");\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert!(deltas(&diff).is_empty());
+    assert!(diff.failures.is_some());
+    let failures = diff.failures.unwrap();
+    assert_eq!(failures.fuzzy_match_failures, 1);
+}
+
+#[test]
+fn test_v4a_noop_diff() {
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "fn main() {".to_string(),
+        old: "    println!(\"Hello\");".to_string(),
+        new: "    println!(\"Hello\");".to_string(), // Same as old
+        post_context: "}".to_string(),
+    }];
+
+    let file_content = "fn main() {\n    println!(\"Hello\");\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert!(deltas(&diff).is_empty());
+    assert!(diff.failures.is_some());
+    let failures = diff.failures.unwrap();
+    assert_eq!(failures.noop_deltas, 1);
+}
+
+#[test]
+fn test_v4a_empty_context() {
+    // Test with no pre or post context
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: String::new(),
+        old: "let x = 1;".to_string(),
+        new: "let x = 2;".to_string(),
+        post_context: String::new(),
+    }];
+
+    let file_content = "let x = 1;";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(
+        deltas(&diff)[0],
+        DiffDelta {
+            replacement_line_range: 1..2,
+            insertion: "let x = 2;".to_string(),
+        }
+    );
+}
+
+#[test]
+fn test_v4a_multiline_old_content() {
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "fn calculate() {".to_string(),
+        old: "    let a = 1;\n    let b = 2;\n    let sum = a + b;".to_string(),
+        new: "    let sum = 3;".to_string(),
+        post_context: "    println!(\"{}\", sum);\n}".to_string(),
+    }];
+
+    let file_content = "fn calculate() {\n    let a = 1;\n    let b = 2;\n    let sum = a + b;\n    println!(\"{}\", sum);\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    assert_eq!(
+        deltas(&diff)[0],
+        DiffDelta {
+            replacement_line_range: 2..5,
+            insertion: "    let sum = 3;".to_string(),
+        }
+    );
+}
+
+#[test]
+fn test_v4a_multiple_hunks() {
+    let hunks = vec![
+        V4AHunk {
+            change_context: vec![],
+            pre_context: "fn first() {".to_string(),
+            old: "    let x = 1;".to_string(),
+            new: "    let x = 10;".to_string(),
+            post_context: "}".to_string(),
+        },
+        V4AHunk {
+            change_context: vec![],
+            pre_context: "fn second() {".to_string(),
+            old: "    let y = 2;".to_string(),
+            new: "    let y = 20;".to_string(),
+            post_context: "}".to_string(),
+        },
+    ];
+
+    let file_content = "fn first() {\n    let x = 1;\n}\n\nfn second() {\n    let y = 2;\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 2);
+    assert_eq!(deltas(&diff)[0].replacement_line_range, 2..3);
+    assert_eq!(deltas(&diff)[0].insertion, "    let x = 10;");
+    assert_eq!(deltas(&diff)[1].replacement_line_range, 6..7);
+    assert_eq!(deltas(&diff)[1].insertion, "    let y = 20;");
+}
+
+#[test]
+fn test_v4a_add_line_with_change_context_no_old() {
+    // Test adding a new line using only change_context to locate position, without old content or pre-context
+    let hunks = vec![V4AHunk {
+        change_context: vec!["class MyClass {".to_string()],
+        pre_context: "".to_string(),
+        old: "".to_string(),
+        new: "    fn new_method() {\n        return 2;\n    }".to_string(),
+        post_context: "    fn existing_method() {".to_string(),
+    }];
+
+    let file_content = "class MyClass {\n    fn existing_method() {\n        return 1;\n    }\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    // The insertion should happen after the change_context line (line 1)
+    assert_eq!(deltas(&diff)[0].replacement_line_range, 2..2);
+    assert_eq!(
+        deltas(&diff)[0].insertion,
+        "    fn new_method() {\n        return 2;\n    }"
+    );
+}
+
+#[test]
+fn test_v4a_add_line_at_start_of_file() {
+    // Test adding a line at the very start of a file
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "".to_string(), // No pre-context - start of file
+        old: "".to_string(),         // No old content
+        new: "// New header comment".to_string(),
+        post_context: "fn main() {".to_string(),
+    }];
+
+    let file_content = "fn main() {\n    println!(\"Hello\");\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    // Should insert at the beginning (line range 1..1 means before line 1)
+    assert_eq!(deltas(&diff)[0].replacement_line_range, 1..1);
+    assert_eq!(deltas(&diff)[0].insertion, "// New header comment");
+}
+
+#[test]
+fn test_v4a_add_line_at_end_of_file() {
+    // Test adding a line at the very end of a file
+    let hunks = vec![V4AHunk {
+        change_context: vec![],
+        pre_context: "fn main() {\n    println!(\"Hello\");\n}".to_string(),
+        old: "".to_string(), // No old content
+        new: "\n// Footer comment".to_string(),
+        post_context: "".to_string(), // No post-context - end of file
+    }];
+
+    let file_content = "fn main() {\n    println!(\"Hello\");\n}";
+    let diff = fuzzy_match_v4a_diffs("test.rs", &hunks, None, file_content);
+
+    assert_eq!(deltas(&diff).len(), 1);
+    // Should insert after the last line (line 3), so insertion point is 4..4
+    assert_eq!(deltas(&diff)[0].replacement_line_range, 4..4);
+    assert_eq!(deltas(&diff)[0].insertion, "\n// Footer comment");
+}
+
+#[test]
+fn test_partial_last_line_in_search_preserves_suffix() {
+    // When a search string ends with a partial line (e.g. "let x = 1;\nlet x" where
+    // "let x" is only a prefix of the actual file line "let x = 2;"), the Jaro-Winkler
+    // fuzzy matcher matches via whole-line windows. The unmatched suffix (" = 2;") from
+    // the file's last matched line must be preserved in the insertion.
+    let file_content = "func foo() {\nlet x = 1;\nlet x = 2;\n}";
+
+    let diffs = [SearchAndReplace {
+        search: "let x = 1;\nlet x".to_string(),
+        replace: "let y = 1;\nlet x".to_string(),
+    }];
+
+    let (deltas, _failures) = fuzzy_match_file_diffs(&diffs, file_content);
+
+    assert_eq!(deltas.len(), 1, "Expected one matched delta");
+    assert_eq!(deltas[0].replacement_line_range, 2..4);
+    // The insertion has the unmatched suffix " = 2;" appended to the last line.
+    assert_eq!(deltas[0].insertion, "let y = 1;\nlet x = 2;");
+
+    // Verify applying the delta produces correct output (no data loss).
+    let file_lines: Vec<&str> = file_content.lines().collect();
+    let range = &deltas[0].replacement_line_range;
+    let mut result = String::new();
+    for line in &file_lines[..range.start - 1] {
+        result.push_str(line);
+        result.push('\n');
+    }
+    result.push_str(&deltas[0].insertion);
+    result.push('\n');
+    for line in &file_lines[range.end - 1..] {
+        result.push_str(line);
+        result.push('\n');
+    }
+    assert_eq!(result, "func foo() {\nlet y = 1;\nlet x = 2;\n}\n");
+}
+
+#[test]
+fn test_search_and_replace_accommodates_none() {
+    let parsed_diff = ParsedDiff::StrReplaceEdit {
+        file: None,
+        search: None,
+        replace: None,
+    };
+    let search_and_replace: Result<SearchAndReplace, ()> = parsed_diff.try_into();
+    assert_eq!(Err(()), search_and_replace);
+
+    let parsed_diff = ParsedDiff::StrReplaceEdit {
+        file: None,
+        search: Some("search".into()),
+        replace: None,
+    };
+    assert_eq!(
+        Ok(SearchAndReplace {
+            search: "search".into(),
+            replace: String::new()
+        }),
+        parsed_diff.try_into()
+    );
+
+    let parsed_diff = ParsedDiff::StrReplaceEdit {
+        file: None,
+        search: None,
+        replace: Some("replace".into()),
+    };
+    assert_eq!(
+        Ok(SearchAndReplace {
+            search: String::new(),
+            replace: "replace".into()
+        }),
+        parsed_diff.try_into()
+    );
+}
+
+/// Test that if a search/replace pair is not a noop, but the overall effect is a noop when applied
+/// to the file contents, we skip the diff.
+#[test]
+fn test_replace_matches_file_content() {
+    let diffs = [SearchAndReplace {
+        search: "1|Hey, there".to_string(),
+        replace: "Hi, there".to_string(),
+    }];
+    let (deltas, errors) = fuzzy_match_file_diffs(&diffs, "Hi, there\nGoodbye, world");
+    assert!(deltas.is_empty());
+    assert_eq!(errors.noop_deltas, 1);
+}
+
+#[test]
+fn test_search_range_greater_than_file_length() {
+    // This should not panic!
+    let r = match_diff(
+        "hey\nthere",
+        Some(14..15),
+        &["hey", "there"],
+        1f64,
+        MakeExactMatch,
+    );
+
+    assert_eq!(r, Some(1..3));
+}
+
+#[test]
+fn test_custom_lines() {
+    assert_eq!(lines("").collect_vec(), vec![""]);
+    assert_eq!(lines("foobar").collect_vec(), vec!["foobar"]);
+    assert_eq!(lines("foo\nbar").collect_vec(), vec!["foo", "bar"]);
+    assert_eq!(lines("foo\nbar\n").collect_vec(), vec!["foo", "bar"]);
+}

crates/ai/src/document.rs

@@ -0,0 +1,58 @@
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
+pub struct AIDocumentId(Uuid);
+
+impl AIDocumentId {
+    pub fn new() -> Self {
+        Self(Uuid::new_v4())
+    }
+}
+
+impl std::fmt::Display for AIDocumentId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl TryFrom<String> for AIDocumentId {
+    type Error = anyhow::Error;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        Ok(Self(Uuid::try_parse(&value)?))
+    }
+}
+
+impl TryFrom<&str> for AIDocumentId {
+    type Error = anyhow::Error;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        Ok(Self(Uuid::try_parse(value)?))
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AIDocumentVersion(pub usize);
+
+impl AIDocumentVersion {
+    #[cfg(feature = "test-util")]
+    pub fn new_for_test(version: usize) -> Self {
+        Self(version)
+    }
+
+    pub fn next(&self) -> Self {
+        Self(self.0 + 1)
+    }
+}
+
+impl std::fmt::Display for AIDocumentVersion {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "v{}", self.0)
+    }
+}
+
+impl Default for AIDocumentVersion {
+    fn default() -> Self {
+        Self(1)
+    }
+}

crates/ai/src/gfm_table.rs

@@ -0,0 +1,242 @@
+use std::iter::Peekable;
+
+use unicode_width::UnicodeWidthStr;
+
+#[derive(Clone, Copy)]
+enum ColumnAlignment {
+    Left,
+    Center,
+    Right,
+}
+
+/// Split a table row into cells, handling escaped pipes (`\|`) as literal pipe characters.
+fn split_cells_escaped(line: &str) -> Vec<String> {
+    let trimmed = line.trim().trim_matches('|');
+    let mut cells = Vec::new();
+    let mut current_cell = String::new();
+    let mut chars = trimmed.chars().peekable();
+
+    while let Some(c) = chars.next() {
+        if c == '\\' && chars.peek() == Some(&'|') {
+            current_cell.push('|');
+            chars.next();
+        } else if c == '|' {
+            cells.push(current_cell.trim().to_string());
+            current_cell = String::new();
+        } else {
+            current_cell.push(c);
+        }
+    }
+    cells.push(current_cell.trim().to_string());
+    cells
+}
+
+/// Returns true if the line looks like a GFM pipe-table separator row,
+/// e.g. `| --- | ---: | :---: |`.
+fn is_gfm_table_separator_row(row: &str) -> bool {
+    let trimmed = row.trim();
+    if trimmed.is_empty() || !trimmed.contains('|') {
+        return false;
+    }
+
+    let mut contains_separator_cell = false;
+    // Split row into individual cells.
+    for cell in trimmed.split('|').map(|c| c.trim()) {
+        if cell.is_empty() {
+            continue;
+        }
+
+        // `:` are used to indicate a column's horizontal alignment (e.g. `:--:` for center).
+        let dashes = cell.trim_matches(':').trim();
+        if dashes.is_empty() {
+            return false;
+        }
+        if !dashes.chars().all(|c| c == '-') {
+            return false;
+        }
+        contains_separator_cell = true;
+    }
+    contains_separator_cell
+}
+
+/// Attempts to parse a GFM table starting from `header_line`.
+///
+/// If the next line in `lines` is a valid GFM separator row, this consumes all
+/// subsequent table rows and returns the raw table lines.
+/// The `should_stop` predicate is called on each candidate row to allow the caller
+/// to halt parsing early (e.g., when encountering a fenced code block).
+///
+/// Returns `None` if `header_line` and the next line don't form a valid table start.
+pub fn maybe_collect_gfm_table_lines<'a, I>(
+    header_line: &str,
+    lines: &mut Peekable<I>,
+    should_stop: impl Fn(&str) -> bool,
+) -> Option<Vec<String>>
+where
+    I: Iterator<Item = &'a str>,
+{
+    let header_trimmed = header_line.trim();
+    let has_leading_or_trailing_pipe =
+        header_trimmed.starts_with('|') || header_trimmed.ends_with('|');
+    let has_at_least_two_pipes = header_trimmed.matches('|').count() >= 2;
+    if !has_leading_or_trailing_pipe || !has_at_least_two_pipes {
+        return None;
+    }
+
+    let separator = lines
+        .next_if(|line| is_gfm_table_separator_row(line))?
+        .to_owned();
+
+    let header_column_count = split_cells_escaped(header_trimmed).len();
+    let separator_column_count = split_cells_escaped(&separator).len();
+    if header_column_count != separator_column_count {
+        return None;
+    }
+
+    let mut table_lines = vec![header_line.to_owned(), separator];
+
+    while let Some(next_line) = lines.peek() {
+        let is_blank = next_line.trim().is_empty();
+        let is_end_of_section = should_stop(next_line);
+        let row_column_count = split_cells_escaped(next_line).len();
+        let has_wrong_column_count = row_column_count != header_column_count;
+        if is_blank || is_end_of_section || has_wrong_column_count {
+            break;
+        }
+        table_lines.push(lines.next().expect("peeked line must exist").to_owned());
+    }
+
+    Some(table_lines)
+}
+
+/// Attempts to parse a GFM table starting from `header_line`.
+///
+/// If the next line in `lines` is a valid GFM separator row, this consumes all
+/// subsequent table rows and returns the formatted table as a `String`.
+/// The `should_stop` predicate is called on each candidate row to allow the caller
+/// to halt parsing early (e.g., when encountering a fenced code block).
+///
+/// Returns `None` if `header_line` and the next line don't form a valid table start.
+pub fn maybe_parse_gfm_table<'a, I>(
+    header_line: &str,
+    lines: &mut Peekable<I>,
+    should_stop: impl Fn(&str) -> bool,
+) -> Option<String>
+where
+    I: Iterator<Item = &'a str>,
+{
+    maybe_collect_gfm_table_lines(header_line, lines, should_stop)
+        .map(|table_lines| format_gfm_table(&table_lines))
+}
+
+/// Formats a GFM table with normalized column widths.
+pub fn format_gfm_table(rows: &[String]) -> String {
+    // A valid GFM table must consist of at least two rows
+    // (a header row and a separator row).
+    if rows.len() < 2 {
+        return rows.join("\n");
+    }
+
+    // Parse all rows into cells, handling leading/trailing pipes
+    let parsed_rows: Vec<Vec<String>> = rows
+        .iter()
+        .map(|row| {
+            let trimmed = row.trim();
+            if trimmed.is_empty() {
+                return vec![];
+            }
+
+            // Split into cells, handling escaped pipes.
+            split_cells_escaped(trimmed)
+        })
+        .collect();
+
+    let num_columns = parsed_rows.first().map_or(0, |r| r.len());
+    if num_columns == 0 {
+        return rows.join("\n");
+    }
+
+    // Calculate max display width for each column
+    // (3 is the minimum width for the separator row).
+    let mut column_widths = vec![3usize; num_columns];
+    for (row_idx, row) in parsed_rows.iter().enumerate() {
+        if row_idx == 1 {
+            continue;
+        }
+        for (col_idx, cell) in row.iter().enumerate() {
+            if col_idx < num_columns {
+                column_widths[col_idx] = column_widths[col_idx].max(cell.width());
+            }
+        }
+    }
+
+    // Parse alignments from separator row
+    let alignments: Vec<ColumnAlignment> = parsed_rows
+        .get(1)
+        .map(|sep_row| {
+            (0..num_columns)
+                .map(|i| {
+                    sep_row.get(i).map_or(ColumnAlignment::Left, |cell| {
+                        let cell = cell.trim();
+                        match (cell.starts_with(':'), cell.ends_with(':')) {
+                            // :---: => center aligned
+                            (true, true) => ColumnAlignment::Center,
+                            // ---: => right aligned
+                            (false, true) => ColumnAlignment::Right,
+                            // :--- or --- => left aligned
+                            _ => ColumnAlignment::Left,
+                        }
+                    })
+                })
+                .collect()
+        })
+        .unwrap_or_else(|| vec![ColumnAlignment::Left; num_columns]);
+
+    // Build formatted rows
+    let mut result = Vec::with_capacity(rows.len());
+    for (row_idx, row) in parsed_rows.iter().enumerate() {
+        let formatted_cells: Vec<String> = (0..num_columns)
+            .map(|col_idx| {
+                let width = column_widths[col_idx];
+                let alignment = alignments[col_idx];
+                if row_idx == 1 {
+                    // Use format padding to generate repeated dashes for the separator rows
+                    // (e.g. `{:-<width$}` pads "-" on the right with `-` chars to reach `width`).
+                    match alignment {
+                        ColumnAlignment::Left => format!("{:-<width$}", "-"),
+                        ColumnAlignment::Right => {
+                            let dashes = width.saturating_sub(1);
+                            format!("{:-<dashes$}:", "-")
+                        }
+                        ColumnAlignment::Center => {
+                            let dashes = width.saturating_sub(2);
+                            format!(":{:-<dashes$}:", "-")
+                        }
+                    }
+                } else {
+                    // Data row: pad manually since format! doesn't account for
+                    // Unicode display width (e.g. emojis are wider than 1 char).
+                    let cell = row.get(col_idx).map_or("", |s| s.as_str());
+                    let display_width = cell.width();
+                    let padding = width.saturating_sub(display_width);
+                    match alignment {
+                        ColumnAlignment::Left => format!("{cell}{:padding$}", ""),
+                        ColumnAlignment::Right => format!("{:padding$}{cell}", ""),
+                        ColumnAlignment::Center => {
+                            let left_pad = padding / 2;
+                            let right_pad = padding - left_pad;
+                            format!("{:left_pad$}{cell}{:right_pad$}", "", "")
+                        }
+                    }
+                }
+            })
+            .collect();
+        result.push(format!("| {} |", formatted_cells.join(" | ")));
+    }
+
+    result.join("\n")
+}
+
+#[cfg(test)]
+#[path = "gfm_table_tests.rs"]
+mod tests;

crates/ai/src/gfm_table_tests.rs

@@ -0,0 +1,90 @@
+use super::format_gfm_table;
+
+#[test]
+fn format_gfm_table_normalizes_column_widths() {
+    let lines = vec![
+        "| Short | Medium Length | This is much longer |".to_owned(),
+        "| --- | --- | --- |".to_owned(),
+        "| A | Hello world | X |".to_owned(),
+    ];
+    let result = format_gfm_table(&lines);
+    let result_lines: Vec<&str> = result.lines().collect();
+
+    assert_eq!(result_lines.len(), 3);
+    // All rows should have the same length due to padding
+    assert_eq!(result_lines[0].len(), result_lines[1].len());
+    assert_eq!(result_lines[1].len(), result_lines[2].len());
+    // Check content is preserved
+    assert!(result_lines[0].contains("Short"));
+    assert!(result_lines[0].contains("Medium Length"));
+    assert!(result_lines[0].contains("This is much longer"));
+    assert!(result_lines[2].contains("A"));
+    assert!(result_lines[2].contains("Hello world"));
+}
+
+#[test]
+fn format_gfm_table_preserves_alignment_markers() {
+    let lines = vec![
+        "| Left | Center | Right |".to_owned(),
+        "| :--- | :---: | ---: |".to_owned(),
+        "| A | B | C |".to_owned(),
+    ];
+    let result = format_gfm_table(&lines);
+    let sep_line = result.lines().nth(1).unwrap();
+
+    // Extract separator cells (trim leading/trailing pipes and split)
+    let cells: Vec<&str> = sep_line
+        .trim()
+        .trim_matches('|')
+        .split('|')
+        .map(|c| c.trim())
+        .collect();
+
+    assert_eq!(cells.len(), 3);
+    // Left alignment: starts with dashes (no leading colon after trimming)
+    assert!(
+        cells[0].starts_with('-'),
+        "Left column should be left-aligned"
+    );
+    // Center alignment: starts and ends with colon
+    assert!(
+        cells[1].starts_with(':') && cells[1].ends_with(':'),
+        "Center column should be center-aligned"
+    );
+    // Right alignment: ends with colon but doesn't start with one
+    assert!(
+        !cells[2].starts_with(':') && cells[2].ends_with(':'),
+        "Right column should be right-aligned"
+    );
+}
+
+#[test]
+fn format_gfm_table_handles_rows_with_fewer_columns() {
+    let lines = vec![
+        "| A | B | C |".to_owned(),
+        "| --- | --- | --- |".to_owned(),
+        "| X |".to_owned(), // Missing columns
+    ];
+    let result = format_gfm_table(&lines);
+    let result_lines: Vec<&str> = result.lines().collect();
+
+    // Should still produce valid output
+    assert_eq!(result_lines.len(), 3);
+    // Last row should be padded to have same structure
+    assert_eq!(result_lines[0].len(), result_lines[2].len());
+}
+
+#[test]
+fn format_gfm_table_handles_empty_cells() {
+    let lines = vec![
+        "| A | | C |".to_owned(),
+        "| --- | --- | --- |".to_owned(),
+        "| | B | |".to_owned(),
+    ];
+    let result = format_gfm_table(&lines);
+
+    // Should produce aligned output with empty cells preserved
+    assert!(result.contains("| A"));
+    assert!(result.contains("| B"));
+    assert!(result.contains("| C"));
+}

crates/ai/src/index/file_outline/mod.rs

@@ -0,0 +1,184 @@
+#![cfg_attr(target_arch = "wasm32", allow(dead_code))]
+
+cfg_if::cfg_if! {
+    if #[cfg(not(target_arch = "wasm32"))] {
+        mod native;
+        pub use native::build_outline;
+    }
+}
+
+use crate::index::{Entry, FileId};
+use ignore::gitignore::Gitignore;
+use std::collections::{HashMap, VecDeque};
+use std::path::PathBuf;
+
+use itertools::Itertools;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct FileSymbols {
+    pub path: String,
+    pub symbols: String,
+}
+
+/// Builds an "outline" of all files and directories under a path.
+#[derive(Debug)]
+pub struct Outline {
+    /// Tree representation of the outlined directory.
+    root: Entry,
+
+    /// Mapping the leaf file nodes to their outline.
+    file_id_to_outline: HashMap<FileId, FileOutline>,
+
+    /// List of gitignore patterns.
+    gitignores: Vec<Gitignore>,
+}
+
+impl Outline {
+    /// Format the outline into a list of text representation with FileContexts.
+    ///
+    /// If `partial_path_segments` is `Some()`, only returns the symbols of files where at least
+    /// one of the segments is contained in the full file path.  [] for partial_paths returns all files.
+    pub fn to_file_symbols(&self, partial_path_segments: Option<&Vec<String>>) -> Vec<FileSymbols> {
+        let mut queue = VecDeque::from([&self.root]);
+        let mut repo_map = Vec::new();
+
+        // Iteratively print the files while perserving their traversal order.
+        while let Some(entry) = queue.pop_front() {
+            match entry {
+                Entry::Directory(directory) => {
+                    queue.extend(&directory.children);
+                }
+                Entry::File(file) => {
+                    let Some(relative_file_path) = file.path.strip_prefix(self.root.path()) else {
+                        continue;
+                    };
+
+                    if partial_path_segments.is_some_and(|paths| {
+                        !paths.is_empty()
+                            && !paths
+                                .iter()
+                                .any(|partial_path| relative_file_path.contains(partial_path))
+                    }) {
+                        continue;
+                    }
+
+                    let mut context = FileSymbols {
+                        path: relative_file_path.to_string(),
+                        symbols: String::new(),
+                    };
+
+                    if let Some(file_outline) = self
+                        .file_id_to_outline
+                        .get(&file.file_id)
+                        .and_then(|outline| outline.to_string())
+                    {
+                        context.symbols = file_outline;
+                    }
+
+                    repo_map.push(context);
+                }
+            }
+        }
+
+        repo_map
+    }
+
+    pub fn to_symbols_by_file(
+        &self,
+        partial_path_segments: Option<&Vec<String>>,
+    ) -> HashMap<PathBuf, FileOutline> {
+        let mut queue = VecDeque::from([&self.root]);
+        let mut file_to_symbols = HashMap::new();
+
+        // Iteratively print the files while perserving their traversal order.
+        while let Some(entry) = queue.pop_front() {
+            match entry {
+                Entry::Directory(directory) => {
+                    queue.extend(&directory.children);
+                }
+                Entry::File(file) => {
+                    let Some(relative_file_path) = file.path.strip_prefix(self.root.path()) else {
+                        continue;
+                    };
+
+                    if partial_path_segments.is_some_and(|paths| {
+                        !paths.is_empty()
+                            && !paths
+                                .iter()
+                                .any(|partial_path| relative_file_path.contains(partial_path))
+                    }) {
+                        continue;
+                    }
+
+                    if let Some(file_outline) = self.file_id_to_outline.get(&file.file_id) {
+                        file_to_symbols
+                            .insert(file.path.to_local_path_lossy(), file_outline.clone());
+                    }
+                }
+            }
+        }
+
+        file_to_symbols
+    }
+
+    pub fn file_count(&self) -> usize {
+        self.file_id_to_outline.len()
+    }
+
+    pub fn gitignores(&self) -> Vec<Gitignore> {
+        self.gitignores.clone()
+    }
+}
+
+/// An identifier symbol in the code file. For now this is just the top-level functions.
+#[derive(Debug, Clone)]
+pub struct Symbol {
+    pub name: String,
+    /// The type prefix to the symbol. This is language specific.
+    /// For example, for a function in rust, this will be "fn". Note that this could be
+    /// empty if the symbol type does not have a prefix (e.g. methods in javascript).
+    pub type_prefix: Option<String>,
+    /// Line comments attached to the symbol.
+    pub comment: Option<Vec<String>>,
+    /// The starting line number of the symbol (1-indexed).
+    pub line_number: usize,
+}
+
+/// Represents the "outline" of a file with all the identifier symbols of interest.
+#[derive(Debug, Clone, Default)]
+pub struct FileOutline {
+    symbols: Option<Vec<Symbol>>,
+}
+
+impl FileOutline {
+    /// Get the symbols from the outline.
+    pub fn symbols(&self) -> Option<&Vec<Symbol>> {
+        self.symbols.as_ref()
+    }
+
+    /// Format the outline into a string.
+    pub fn to_string(&self) -> Option<String> {
+        Some(
+            self.symbols
+                .as_ref()?
+                .iter()
+                .map(|identifier| {
+                    let symbol = match &identifier.type_prefix {
+                        Some(type_prefix) => format!(
+                            "  {} {} (line {})",
+                            type_prefix, identifier.name, identifier.line_number
+                        ),
+                        None => format!("  {} (line {})", identifier.name, identifier.line_number),
+                    };
+                    match &identifier.comment {
+                        Some(comment) => {
+                            format!("  {}\n{}", comment.join("\n  "), symbol)
+                        }
+                        None => symbol,
+                    }
+                })
+                .join("\n"),
+        )
+    }
+}

crates/ai/src/index/file_outline/native.rs

@@ -0,0 +1,531 @@
+use futures::channel::oneshot;
+use ignore::gitignore::Gitignore;
+use rayon::prelude::*;
+use repo_metadata::entry::is_file_parsable;
+use repo_metadata::RepositoryUpdate;
+use std::collections::HashMap;
+use std::{fs, path::Path};
+
+use anyhow::anyhow;
+use arborium::tree_sitter::{Parser, Query, QueryCursor, Tree};
+use itertools::Itertools;
+use streaming_iterator::StreamingIterator;
+use syntax_tree::TextSlice;
+
+use crate::index::file_outline::{FileOutline, Outline, Symbol};
+use crate::index::THREADPOOL;
+use crate::index::{Entry, FileId, FileMetadata};
+use repo_metadata::entry::IgnoredPathStrategy;
+
+cfg_if::cfg_if! {
+    if #[cfg(feature = "local_fs")] {
+        use crate::index::matches_gitignores;
+    }
+}
+
+/// Given a repo path, try to build its outline. An outline is a list of all its files and the symbols
+/// of interest from each file.
+pub async fn build_outline(
+    path: &Path,
+    max_num_files_limit: Option<usize>,
+) -> anyhow::Result<Outline> {
+    const MAX_DEPTH: usize = 200;
+    let mut gitignores = vec![];
+
+    // Add global gitignore, if it exists
+    let (global_gitignore, _) = Gitignore::global();
+    if !global_gitignore.is_empty() {
+        gitignores.push(global_gitignore);
+    }
+
+    let gitignore_path = path.join(".gitignore");
+    if gitignore_path.exists() {
+        let (gitignore, _) = Gitignore::new(gitignore_path);
+        gitignores.push(gitignore);
+    }
+
+    // First traverse the repo path to retrieve all files we want to parse.
+    let mut files = Vec::new();
+    let mut remaining_file_quotas = max_num_files_limit;
+    let entry = Entry::build_tree(
+        path,
+        &mut files,
+        &mut gitignores,
+        remaining_file_quotas.as_mut(),
+        MAX_DEPTH,
+        0,
+        &IgnoredPathStrategy::Exclude, // override_ignore_for_files
+    )?;
+
+    let (sender, receiver) = oneshot::channel();
+
+    let Some(pool) = THREADPOOL.as_ref() else {
+        return Err(anyhow!("No threadpool exists for outline generation."));
+    };
+
+    pool.spawn(move || {
+        // Parse each file in parallel. Note that we have to fold and then reduce given the parallelization.
+        let result = pool.install(|| {
+            files
+                .par_iter()
+                .map(|metadata| {
+                    let outline = parse_file_outline(&metadata.path.to_local_path_lossy())
+                        .ok()
+                        .unwrap_or_default();
+
+                    (metadata.file_id, outline)
+                })
+                .collect::<HashMap<_, _>>()
+        });
+
+        if let Err(e) = sender.send(result) {
+            log::error!("Could not send result of outline generation to background thread. {e:?}")
+        }
+    });
+
+    let file_id_to_outline = receiver.await?;
+
+    Ok(Outline {
+        root: entry,
+        file_id_to_outline,
+        gitignores,
+    })
+}
+
+impl Outline {
+    /// Update this outline in-place with a set of changed files. This is asynchronous because it
+    /// requires re-parsing modified files.
+    pub async fn update(&mut self, outline_update: RepositoryUpdate) {
+        let RepositoryUpdate {
+            added,
+            modified,
+            deleted,
+            moved,
+            ..
+        } = outline_update;
+
+        let mut files_metadata = vec![];
+        let mut files_metadata_to_remove = vec![];
+
+        // Extract paths from TargetFile for removal, filtering out gitignored files
+        for target_file in deleted
+            .into_iter()
+            .chain(moved.values().cloned())
+            .filter(|target_file| !target_file.is_ignored)
+        {
+            if let Some(metadata) = self.root.remove(&target_file.path) {
+                files_metadata_to_remove.push(metadata);
+            }
+        }
+
+        // Extract paths from TargetFile for addition, filtering out gitignored files
+        for target_file in added
+            .into_iter()
+            .chain(modified.into_iter())
+            .chain(moved.keys().cloned())
+            .filter(|target_file| !target_file.is_ignored)
+        {
+            if let Some(file_metadata) = self.find_or_insert_path_to_file_tree(&target_file.path) {
+                files_metadata.push(file_metadata.clone());
+            }
+        }
+
+        for metadata in &files_metadata_to_remove {
+            self.file_id_to_outline.remove(&metadata.file_id);
+        }
+
+        if let Some(updated_outlines) = parse_symbols_for_files(files_metadata).await {
+            self.file_id_to_outline.extend(updated_outlines);
+        }
+    }
+
+    /// Returns the `FileMetadata` for the file corresponding to the given target path.
+    ///
+    /// If the target path corresponds to a directory, returns `None`.
+    fn find_or_insert_path_to_file_tree(&mut self, target_path: &Path) -> Option<&FileMetadata> {
+        match &mut self.root {
+            Entry::Directory(directory) => {
+                let dir_local = directory.path.to_local_path_lossy();
+                if target_path.strip_prefix(&dir_local).is_err() {
+                    // Target is not descendant of the repo.
+                    return None;
+                }
+
+                // Get all the ancestors between the target path and the directory, including the
+                // target path itself.
+                let ancestors_between_target_and_directory = std::iter::once(target_path)
+                    .chain(
+                        target_path
+                            .ancestors()
+                            .take_while(|ancestor| *ancestor != dir_local.as_path()),
+                    )
+                    .collect_vec();
+
+                // Iterate over the ancestors in reverse order, starting from the ancestor that is
+                // the child of `directory`. We get or insert the entry corresponding to each of
+                // those target ancestors, and continue the iteration if that entry is a directory.
+                // At the end of the iteration we'll have reached the target path.
+                let mut current_parent = directory;
+                for ancestor in ancestors_between_target_and_directory.iter().rev() {
+                    if matches_gitignores(
+                        ancestor,
+                        ancestor.is_dir(),
+                        &self.gitignores,
+                        false, /* check_ancestors */
+                    ) || ancestor.ends_with(".git")
+                    {
+                        // Short-circuit if an ancestor is ignored.
+                        return None;
+                    }
+                    match current_parent.find_or_insert_child(ancestor) {
+                        Some(Entry::File(file_metadata)) => {
+                            // If this entry is a file, we've reached the target path -- files can't
+                            // have children!
+                            return Some(&*file_metadata);
+                        }
+                        Some(Entry::Directory(directory)) => {
+                            current_parent = directory;
+                        }
+                        None => return None,
+                    }
+                }
+                None
+            }
+            Entry::File(_) => {
+                log::error!("File tree root shouldn't be a file node");
+                None
+            }
+        }
+    }
+}
+
+/// Parse file symbols in parallel. This uses the [shared Rayon file-parsing pool](THREADPOOL),
+/// but is `async` because it MUST NOT be called from the main thread.
+async fn parse_symbols_for_files(files: Vec<FileMetadata>) -> Option<HashMap<FileId, FileOutline>> {
+    let pool = THREADPOOL.as_ref()?;
+
+    let (tx, rx) = oneshot::channel();
+
+    pool.install(move || {
+        rayon::spawn(move || {
+            // Parse each file in parallel. Note that we have to fold and then reduce given the parallelization.
+            let result = files
+                .par_iter()
+                .map(|metadata| {
+                    let outline = parse_file_outline(&metadata.path.to_local_path_lossy())
+                        .ok()
+                        .unwrap_or_default();
+
+                    (metadata.file_id, outline)
+                })
+                .collect::<HashMap<_, _>>();
+            let _ = tx.send(result);
+        });
+    });
+
+    rx.await.ok()
+}
+
+/// Given the path of a file, try to construct its outline.
+fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
+    if !is_file_parsable(path)? {
+        return Err(anyhow!("File exceeds max file size limit for parsing"));
+    }
+    let Some(language) = languages::language_by_filename(path) else {
+        return Err(anyhow!("Language unsupported for file {:?}", path));
+    };
+    let content = fs::read_to_string(path)?;
+
+    let mut parser = Parser::new();
+    parser.set_language(&language.grammar)?;
+    let Some(tree) = parser.parse(&content, None) else {
+        return Err(anyhow!("Couldn't parse AST"));
+    };
+    let symbols = language.symbols_query.as_ref().map(|query| {
+        get_symbols(query, &tree, &content)
+            .into_iter()
+            .map(|(fn_name, type_prefix, comments, line_number)| Symbol {
+                name: fn_name.to_owned(),
+                type_prefix: type_prefix.map(String::from),
+                comment: if comments.is_empty() {
+                    None
+                } else {
+                    Some(comments.into_iter().map(String::from).collect())
+                },
+                line_number,
+            })
+            .collect_vec()
+    });
+
+    drop(tree);
+    drop(parser);
+
+    // Release extra unused memory from malloc to the system.  For some
+    // reason, the memory obtained by the allocator is often not released
+    // back to the OS after we're done with it, resulting in high memory
+    // usage (from the perspective of the OS, though not from the perspective
+    // of the allocator).
+    //
+    // See: https://github.com/tree-sitter/tree-sitter/issues/3129
+    #[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
+    unsafe {
+        nix::libc::malloc_trim(0);
+    }
+
+    Ok(FileOutline { symbols })
+}
+
+/// Given the content of a file, return all the symbols of interest.
+fn get_symbols<'a>(
+    query: &'a Query,
+    tree: &Tree,
+    file_content: &'a String,
+) -> Vec<(&'a str, Option<&'a str>, Vec<&'a str>, usize)> {
+    struct PendingComment<'a> {
+        lines: Vec<&'a str>,
+        last_line_number: usize,
+    }
+    let mut cursor = QueryCursor::new();
+    let capture_names = query.capture_names();
+    let mut captures = cursor.captures(query, tree.root_node(), TextSlice(file_content.as_bytes()));
+
+    let mut symbols = vec![];
+    let mut comment: Option<PendingComment> = None;
+    while let Some(matches) = captures.next() {
+        for cap in matches.0.captures {
+            let capture_name = capture_names.get(cap.index as usize);
+            let matched_content =
+                &file_content[cap.node.byte_range().start..cap.node.byte_range().end];
+            let line_number = cap.node.range().start_point.row;
+            match capture_name {
+                Some(name) if *name == "comment" => match comment.as_mut() {
+                    Some(pending_comment)
+                        if pending_comment.last_line_number + 1 == line_number =>
+                    {
+                        pending_comment.lines.push(matched_content.trim());
+                        pending_comment.last_line_number = line_number;
+                    }
+                    _ => {
+                        comment = Some(PendingComment {
+                            lines: vec![matched_content.trim()],
+                            last_line_number: line_number,
+                        })
+                    }
+                },
+                _ => {
+                    let comments = match comment.take() {
+                        Some(pending_comment)
+                            if pending_comment.last_line_number + 1 == line_number =>
+                        {
+                            pending_comment.lines
+                        }
+                        _ => vec![],
+                    };
+                    let type_prefix = capture_name.and_then(|s| s.split(".").nth(1));
+                    symbols.push((matched_content, type_prefix, comments, line_number + 1));
+                    // Convert to 1-indexed
+                }
+            }
+        }
+    }
+
+    symbols
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Write;
+    use std::path::PathBuf;
+    use tempfile::TempDir;
+
+    fn create_test_file(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
+        let file_path = dir.path().join(filename);
+        let mut file = File::create(&file_path).unwrap();
+        file.write_all(content.as_bytes()).unwrap();
+        file_path
+    }
+
+    #[test]
+    fn test_parse_comments() {
+        let temp_dir = TempDir::new().unwrap();
+        let content = r#"
+/// This is a struct for NewFunc
+struct NewFunc {
+    a: str,
+}
+
+// Hello
+// World
+fn first_function() {
+    println!("First");
+}
+
+impl NewFunc {
+    fn second_function() {
+        println!("Second");
+    }
+}
+"#;
+        let file_path = create_test_file(&temp_dir, "multiple.rs", content);
+
+        let outline = parse_file_outline(&file_path).unwrap();
+        let symbols = outline.symbols.unwrap();
+        assert_eq!(symbols[0].name, "NewFunc");
+        assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
+        assert_eq!(
+            symbols[0].comment,
+            Some(vec!["/// This is a struct for NewFunc".to_owned()])
+        );
+        assert_eq!(symbols[0].line_number, 3); // struct NewFunc is on line 3
+        assert_eq!(symbols[1].name, "first_function");
+        assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
+        assert_eq!(symbols[1].line_number, 9); // first_function is on line 9
+        assert_eq!(symbols[2].name, "second_function");
+        assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
+        assert_eq!(symbols[2].line_number, 14); // second_function is on line 14
+    }
+
+    #[test]
+    fn test_parse_multiple_languages() {
+        let temp_dir = TempDir::new().unwrap();
+        let content = r#"
+struct NewFunc {
+    a: str,
+}
+
+fn first_function() {
+    println!("First");
+}
+
+impl NewFunc {
+    fn second_function() {
+        println!("Second");
+    }
+}
+"#;
+        let file_path = create_test_file(&temp_dir, "multiple.rs", content);
+
+        let outline = parse_file_outline(&file_path).unwrap();
+        let symbols = outline.symbols.unwrap();
+        assert_eq!(symbols.len(), 3);
+        assert_eq!(symbols[0].name, "NewFunc");
+        assert_eq!(symbols[0].type_prefix, Some("struct".to_owned()));
+        assert_eq!(symbols[1].name, "first_function");
+        assert_eq!(symbols[1].type_prefix, Some("fn".to_owned()));
+        assert_eq!(symbols[2].name, "second_function");
+        assert_eq!(symbols[2].type_prefix, Some("fn".to_owned()));
+
+        // Test parsing Python code with multiple symbol definitions
+        // This verifies parsing of:
+        // - Regular function definitions (def keyword)
+        // - Class definitions (class keyword)
+        // - Method definitions within a class (def keyword)
+        let python_content = r#"
+def first_function():
+    print("First")
+
+class TestClass:
+    def __init__(self):
+        pass
+
+    def class_method(self):
+        print("Method")
+
+def second_function():
+    print("Second")
+"#;
+        let file_path = create_test_file(&temp_dir, "multiple.py", python_content);
+        let outline = parse_file_outline(&file_path).unwrap();
+        let symbols = outline.symbols.unwrap();
+        assert_eq!(symbols.len(), 5);
+        assert_eq!(symbols[0].name, "first_function");
+        assert_eq!(symbols[0].type_prefix, Some("def".to_owned()));
+        assert_eq!(symbols[1].name, "TestClass");
+        assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
+        assert_eq!(symbols[2].name, "__init__");
+        assert_eq!(symbols[2].type_prefix, Some("def".to_owned()));
+        assert_eq!(symbols[3].name, "class_method");
+        assert_eq!(symbols[3].type_prefix, Some("def".to_owned()));
+        assert_eq!(symbols[4].name, "second_function");
+        assert_eq!(symbols[4].type_prefix, Some("def".to_owned()));
+
+        // Test parsing JavaScript code with multiple symbol definitions
+        // This verifies parsing of:
+        // - Function declarations
+        // - Class declarations
+        // - Method definitions
+        // - Arrow functions assigned to variables
+        let js_content = r#"
+function regularFunction() {
+    console.log('Regular function');
+}
+
+class TestClass {
+    constructor() {
+        this.value = 42;
+    }
+
+    classMethod() {
+        return this.value;
+    }
+}
+"#;
+        let file_path = create_test_file(&temp_dir, "multiple.js", js_content);
+        let outline = parse_file_outline(&file_path).unwrap();
+        let symbols = outline.symbols.unwrap();
+        assert_eq!(symbols.len(), 4);
+        assert_eq!(symbols[0].name, "regularFunction");
+        assert_eq!(symbols[0].type_prefix, Some("function".to_owned()));
+        assert_eq!(symbols[1].name, "TestClass");
+        assert_eq!(symbols[1].type_prefix, Some("class".to_owned()));
+        assert_eq!(symbols[2].name, "constructor");
+        assert_eq!(symbols[2].type_prefix, None);
+        assert_eq!(symbols[3].name, "classMethod");
+        assert_eq!(symbols[3].type_prefix, None);
+
+        // Test parsing Go code with multiple symbol definitions
+        // This verifies parsing of:
+        // - Function definitions (func keyword)
+        // - Type definitions (struct, interface)
+        // - Method definitions (func with receiver)
+        let go_content = r#"
+package main
+
+func mainFunction() {
+    fmt.Println("Main function")
+}
+
+type TestStruct struct {
+    field string
+}
+
+func (t *TestStruct) structMethod() string {
+    return t.field
+}
+
+type TestInterface interface {
+    InterfaceMethod() string
+}
+
+func helperFunction() {
+    fmt.Println("Helper function")
+}
+"#;
+        let file_path = create_test_file(&temp_dir, "multiple.go", go_content);
+        let outline = parse_file_outline(&file_path).unwrap();
+        let symbols = outline.symbols.unwrap();
+        assert_eq!(symbols.len(), 5);
+        assert_eq!(symbols[0].name, "mainFunction");
+        assert_eq!(symbols[0].type_prefix, Some("func".to_owned()));
+        assert_eq!(symbols[1].name, "TestStruct");
+        assert_eq!(symbols[1].type_prefix, Some("type".to_owned()));
+        assert_eq!(symbols[2].name, "structMethod");
+        assert_eq!(symbols[2].type_prefix, Some("func".to_owned()));
+        assert_eq!(symbols[3].name, "TestInterface");
+        assert_eq!(symbols[3].type_prefix, Some("type".to_owned()));
+        assert_eq!(symbols[4].name, "helperFunction");
+        assert_eq!(symbols[4].type_prefix, Some("func".to_owned()));
+    }
+}

crates/ai/src/index/full_source_code_embedding/changed_files.rs

@@ -0,0 +1,50 @@
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+#[derive(Debug, Default, Clone)]
+pub(super) struct ChangedFiles {
+    pub(super) deletions: HashSet<PathBuf>,
+    pub(super) upsertions: HashSet<PathBuf>,
+}
+
+impl ChangedFiles {
+    pub(super) fn is_empty(&self) -> bool {
+        self.deletions.is_empty() && self.upsertions.is_empty()
+    }
+
+    pub(super) fn deletions(&self) -> &HashSet<PathBuf> {
+        &self.deletions
+    }
+
+    /// Merges a subsequent set of file changes into the current set.
+    pub(super) fn merge_subsequent(&mut self, mut subsequent_changes: Self) {
+        for path in subsequent_changes.deletions.drain() {
+            if self.upsertions.contains(&path) {
+                self.upsertions.remove(&path);
+            }
+            self.deletions.insert(path);
+        }
+
+        for path in subsequent_changes.upsertions.drain() {
+            if self.deletions.contains(&path) {
+                self.deletions.remove(&path);
+            }
+            self.upsertions.insert(path);
+        }
+    }
+
+    // Add paths to this changed files set based on whether they currently exist on the file system.
+    pub(super) async fn add_paths(&mut self, paths: impl IntoIterator<Item = PathBuf>) {
+        for path in paths {
+            if path.exists() {
+                self.upsertions.insert(path);
+            } else {
+                self.deletions.insert(path);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "changed_files_test.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/changed_files_test.rs

@@ -0,0 +1,251 @@
+use super::*;
+use std::path::PathBuf;
+
+// Helper function to create a PathBuf from a string
+fn pb(path: &str) -> PathBuf {
+    PathBuf::from(path)
+}
+
+#[test]
+fn test_basic_merge_non_conflicting() {
+    // Initial: delete set {a}, upsert set {b}
+    let mut changes1 = ChangedFiles::default();
+    changes1.deletions.insert(pb("a"));
+    changes1.upsertions.insert(pb("b"));
+
+    // Later changes: delete set {c}, upsert set {d}
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("c"));
+    changes2.upsertions.insert(pb("d"));
+
+    // Merge changes
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {a, c}, upsertions {b, d}
+    assert_eq!(changes1.deletions.len(), 2);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert!(changes1.deletions.contains(&pb("c")));
+
+    assert_eq!(changes1.upsertions.len(), 2);
+    assert!(changes1.upsertions.contains(&pb("b")));
+    assert!(changes1.upsertions.contains(&pb("d")));
+}
+
+#[test]
+fn test_delete_then_upsert() {
+    // Initial: delete set {file1}
+    let mut changes1 = ChangedFiles::default();
+    changes1.deletions.insert(pb("file1"));
+
+    // Later changes: upsert set {file1}
+    let mut changes2 = ChangedFiles::default();
+    changes2.upsertions.insert(pb("file1"));
+
+    // Merge changes
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {}, upsertions {file1}
+    assert_eq!(changes1.deletions.len(), 0);
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("file1")));
+}
+
+#[test]
+fn test_upsert_then_delete() {
+    // Initial: upsert set {file1}
+    let mut changes1 = ChangedFiles::default();
+    changes1.upsertions.insert(pb("file1"));
+
+    // Later changes: delete set {file1}
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("file1"));
+
+    // Merge changes
+    changes1.merge_subsequent(changes2);
+
+    // Expected: upsertions {}, deletions {file1}
+    assert_eq!(changes1.upsertions.len(), 0);
+    assert_eq!(changes1.deletions.len(), 1);
+    assert!(changes1.deletions.contains(&pb("file1")));
+}
+
+#[test]
+fn test_delete_then_delete() {
+    // Initial: delete set {file1}
+    let mut changes1 = ChangedFiles::default();
+    changes1.deletions.insert(pb("file1"));
+
+    // Later changes: delete set {file1} again
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("file1"));
+
+    // Merge changes
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {file1}, upsertions {}
+    assert_eq!(changes1.deletions.len(), 1);
+    assert!(changes1.deletions.contains(&pb("file1")));
+    assert_eq!(changes1.upsertions.len(), 0);
+}
+
+#[test]
+fn test_upsert_then_upsert() {
+    // Initial: upsert set {file1}
+    let mut changes1 = ChangedFiles::default();
+    changes1.upsertions.insert(pb("file1"));
+
+    // Later changes: upsert set {file1} again
+    let mut changes2 = ChangedFiles::default();
+    changes2.upsertions.insert(pb("file1"));
+
+    // Merge changes
+    changes1.merge_subsequent(changes2);
+
+    // Expected: upsertions {file1}, deletions {}
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("file1")));
+    assert_eq!(changes1.deletions.len(), 0);
+}
+
+#[test]
+fn test_empty_sets() {
+    // Test 1: Empty merged into populated
+    let mut changes1 = ChangedFiles::default();
+    changes1.upsertions.insert(pb("file1"));
+    changes1.deletions.insert(pb("file2"));
+
+    let changes2 = ChangedFiles::default();
+
+    changes1.merge_subsequent(changes2);
+
+    // Should remain unchanged
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("file1")));
+    assert_eq!(changes1.deletions.len(), 1);
+    assert!(changes1.deletions.contains(&pb("file2")));
+
+    // Test 2: Populated merged into empty
+    let mut changes3 = ChangedFiles::default();
+
+    let mut changes4 = ChangedFiles::default();
+    changes4.upsertions.insert(pb("file3"));
+    changes4.deletions.insert(pb("file4"));
+
+    changes3.merge_subsequent(changes4);
+
+    // Should take all changes
+    assert_eq!(changes3.upsertions.len(), 1);
+    assert!(changes3.upsertions.contains(&pb("file3")));
+    assert_eq!(changes3.deletions.len(), 1);
+    assert!(changes3.deletions.contains(&pb("file4")));
+}
+
+#[test]
+fn test_multiple_sequential_merges() {
+    // Initial: upsert set {a}, delete set {b}
+    let mut changes1 = ChangedFiles::default();
+    changes1.upsertions.insert(pb("a"));
+    changes1.deletions.insert(pb("b"));
+
+    // First merge: delete a, delete c, upsert b
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("a"));
+    changes2.deletions.insert(pb("c"));
+    changes2.upsertions.insert(pb("b"));
+
+    // Second merge: upsert c, delete d
+    let mut changes3 = ChangedFiles::default();
+    changes3.upsertions.insert(pb("c"));
+    changes3.deletions.insert(pb("d"));
+
+    // Apply first merge
+    changes1.merge_subsequent(changes2);
+
+    // After first merge:
+    // Expected: upsertions {b}, deletions {a, c}
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("b")));
+    assert_eq!(changes1.deletions.len(), 2);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert!(changes1.deletions.contains(&pb("c")));
+
+    // Apply second merge
+    changes1.merge_subsequent(changes3);
+
+    // After second merge:
+    // Expected: upsertions {b, c}, deletions {a, d}
+    assert_eq!(changes1.upsertions.len(), 2);
+    assert!(changes1.upsertions.contains(&pb("b")));
+    assert!(changes1.upsertions.contains(&pb("c")));
+    assert_eq!(changes1.deletions.len(), 2);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert!(changes1.deletions.contains(&pb("d")));
+}
+
+#[test]
+fn test_rename_then_delete() {
+    // Initial: rename {a -> b}
+    let mut changes1 = ChangedFiles::default();
+    changes1.deletions.insert(pb("a"));
+    changes1.upsertions.insert(pb("b"));
+
+    // Later changes: delete {b}
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("b"));
+
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {a, b}, upsertions {}
+    // Note that we don't know whether b had any prior content,
+    // so we can't assume it was a rename.
+    assert_eq!(changes1.deletions.len(), 2);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert!(changes1.deletions.contains(&pb("b")));
+    assert_eq!(changes1.upsertions.len(), 0);
+}
+
+#[test]
+fn test_upsert_then_rename() {
+    // Initial: upsert {a}
+    let mut changes1 = ChangedFiles::default();
+    changes1.upsertions.insert(pb("a"));
+
+    // Later changes: rename {a -> b}
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("a"));
+    changes2.upsertions.insert(pb("b"));
+
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {a}, upsertions {b}
+    // Note that we don't know whether a had any prior content,
+    // so we can't assume it was a rename.
+    assert_eq!(changes1.deletions.len(), 1);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("b")));
+}
+
+#[test]
+fn test_rename_then_rename() {
+    // Initial: rename {a -> b}
+    let mut changes1 = ChangedFiles::default();
+    changes1.deletions.insert(pb("a"));
+    changes1.upsertions.insert(pb("b"));
+
+    // Later changes: rename {b -> c}
+    let mut changes2 = ChangedFiles::default();
+    changes2.deletions.insert(pb("b"));
+    changes2.upsertions.insert(pb("c"));
+
+    changes1.merge_subsequent(changes2);
+
+    // Expected: deletions {a, b}, upsertions {c}
+    // Note that we don't know whether b had any prior content,
+    // so we can't assume it was a rename.
+    assert_eq!(changes1.deletions.len(), 2);
+    assert!(changes1.deletions.contains(&pb("a")));
+    assert!(changes1.deletions.contains(&pb("b")));
+    assert_eq!(changes1.upsertions.len(), 1);
+    assert!(changes1.upsertions.contains(&pb("c")));
+}

crates/ai/src/index/full_source_code_embedding/chunker.rs

@@ -0,0 +1,109 @@
+use std::path::Path;
+
+use string_offset::ByteOffset;
+
+mod naive;
+#[cfg(not(target_family = "wasm"))]
+mod semantic;
+
+/// Number of lines per chunk when chunking naively. While there's no guarantee
+/// that this is below the token limit of the embedding model used on the server,
+/// this should give us more than enough buffer.
+const LINES_PER_CHUNK: usize = 200;
+
+/// The average number of characters per line.
+const AVG_CHAR_PER_LINE: usize = 60;
+
+/// Compute the max byte per chunk based on the average number of characters per line. We assume code is mostly ASCII,
+/// which is why this max chunk makes sense even if we're using bytes instead of characters as our unit of chunking.
+const MAX_BYTES_PER_CHUNK: usize = LINES_PER_CHUNK * AVG_CHAR_PER_LINE;
+
+/// A code fragment with line range information.
+#[derive(Debug, Clone)]
+pub struct Fragment<'a> {
+    /// The content of the fragment.
+    pub content: &'a str,
+    /// Start line number (inclusive).
+    pub start_line: usize,
+    /// End line number (inclusive).
+    pub end_line: usize,
+    /// The start byte index of the fragment in the original source code.
+    pub start_byte_index: ByteOffset,
+    /// The end byte index of the fragment (exclusive) in the original source code.
+    pub end_byte_index: ByteOffset,
+    /// File path of the fragment.
+    pub file_path: &'a Path,
+}
+
+impl<'a> Fragment<'a> {
+    fn size(&self) -> usize {
+        self.content.len()
+    }
+
+    fn append(&mut self, other: &Fragment<'a>, content: &'a str) {
+        self.end_line = other.end_line;
+        self.end_byte_index = other.end_byte_index;
+        self.content = &content[self.start_byte_index.as_usize()..other.end_byte_index.as_usize()];
+    }
+}
+
+/// Coalesce small fragments into larger ones that still respect the `max_bytes_per_chunk`.
+/// Treesitter often produces small fragments that splits function names from the actual function body,
+/// we iterate in reverse to coalesce these chunks into fragments that are more meaningful.
+fn coalesce_fragments<'a>(
+    fragments: impl DoubleEndedIterator<Item = Fragment<'a>>,
+    code: &'a str,
+    max_bytes_per_chunk: usize,
+) -> Vec<Fragment<'a>> {
+    fragments
+        .rev()
+        .fold(
+            Vec::new(),
+            |mut acc: Vec<Fragment<'a>>, mut fragment| match acc.last_mut() {
+                Some(last_item) => {
+                    let new_fragment_size = code
+                        [fragment.start_byte_index.as_usize()..last_item.end_byte_index.as_usize()]
+                        .len();
+                    if new_fragment_size <= max_bytes_per_chunk {
+                        fragment.append(last_item, code);
+                        *last_item = fragment;
+                    } else {
+                        acc.push(fragment);
+                    }
+                    acc
+                }
+                None => {
+                    acc.push(fragment);
+                    acc
+                }
+            },
+        )
+        .into_iter()
+        .rev()
+        .collect()
+}
+
+/// Chunks code into an ordered list of fragments.
+///
+/// The code is chunked "semantically" using treesitter.
+/// If we are unable to generate semantic chunks for any reason, fragments are naively chunked by
+/// lines.
+pub fn chunk_code<'a>(code: &'a str, path: &'a Path) -> Vec<Fragment<'a>> {
+    if let Some(fragments) = try_chunk_code_semantically(code, path) {
+        return fragments;
+    }
+    naive::chunk_code(code, path, MAX_BYTES_PER_CHUNK, LINES_PER_CHUNK)
+}
+
+/// Attempts to chunk code semantically, returning [`None`] if the code
+/// could not be chunked for any reason.
+#[cfg(not(target_family = "wasm"))]
+fn try_chunk_code_semantically<'a>(code: &'a str, path: &'a Path) -> Option<Vec<Fragment<'a>>> {
+    let language = languages::language_by_filename(path)?;
+    semantic::chunk_code(code, path, MAX_BYTES_PER_CHUNK, &language.grammar).ok()
+}
+
+#[cfg(target_family = "wasm")]
+fn try_chunk_code_semantically<'a>(_code: &'a str, _path: &'a Path) -> Option<Vec<Fragment<'a>>> {
+    None
+}

crates/ai/src/index/full_source_code_embedding/chunker/naive.rs

@@ -0,0 +1,108 @@
+use crate::index::full_source_code_embedding::chunker::{coalesce_fragments, Fragment};
+use itertools::Itertools;
+use line_span::{LineSpan, LineSpans};
+use std::path::Path;
+
+/// Chunks the given file into [`Fragment`]s. Each chunk is at most `num_lines_per_chunk` lines long, and contains at most `max_bytes_per_chunk` bytes.
+pub(super) fn chunk_code<'a>(
+    code: &'a str,
+    path: &'a Path,
+    max_bytes_per_chunk: usize,
+    num_lines_per_chunk: usize,
+) -> Vec<Fragment<'a>> {
+    let lines = code.line_spans().enumerate().collect_vec();
+    let chunks = lines.chunks(num_lines_per_chunk);
+
+    chunks
+        .into_iter()
+        .flat_map(|chunk| {
+            let (start_line, start_range) = chunk[0];
+            let (end_line, end_range) =
+                chunk.last().expect("Chunks must have at least one element");
+
+            if (end_range.end() - start_range.start()) > max_bytes_per_chunk {
+                let chunked_fragments = chunk.iter().flat_map(|(line, line_span)| {
+                    chunk_line_by_bytes(code, path, max_bytes_per_chunk, *line, line_span)
+                });
+
+                return coalesce_fragments(chunked_fragments, code, max_bytes_per_chunk);
+            }
+
+            vec![Fragment {
+                content: &code[start_range.start()..end_range.end()],
+                start_line,
+                end_line: *end_line,
+                file_path: path,
+                start_byte_index: start_range.start().into(),
+                end_byte_index: end_range.end().into(),
+            }]
+        })
+        .collect()
+}
+
+/// Chunks the line represented by `line_span` into multiple fragments if it exceeds `max_bytes_per_chunk`.
+fn chunk_line_by_bytes<'a>(
+    code: &'a str,
+    path: &'a Path,
+    max_bytes_per_chunk: usize,
+    line_number: usize,
+    line_span: &LineSpan<'a>,
+) -> Vec<Fragment<'a>> {
+    let line_start = line_span.start();
+    let line_end = line_span.end();
+    let line_content = &code[line_start..line_end];
+    let line_length = line_end - line_start;
+
+    // If the line is smaller than max_bytes_per_chunk, return it as a single fragment
+    if line_length <= max_bytes_per_chunk {
+        return vec![Fragment {
+            content: line_content,
+            start_line: line_number,
+            end_line: line_number,
+            file_path: path,
+            start_byte_index: line_start.into(),
+            end_byte_index: line_end.into(),
+        }];
+    }
+
+    // Otherwise, split the line into multiple fragments
+    let mut fragments = Vec::new();
+    let mut current_start = line_start;
+
+    while current_start < line_end {
+        let remaining_bytes = line_end - current_start;
+        let chunk_size = std::cmp::min(remaining_bytes, max_bytes_per_chunk);
+        let mut chunk_end = current_start + chunk_size;
+
+        // Ensure chunk_end is on a UTF-8 character boundary
+        while chunk_end > current_start && !code.is_char_boundary(chunk_end) {
+            chunk_end -= 1;
+        }
+
+        // If we couldn't find a valid boundary within reasonable distance,
+        // move forward to the next character boundary instead
+        if chunk_end <= current_start {
+            chunk_end = current_start + chunk_size;
+            while chunk_end < line_end && !code.is_char_boundary(chunk_end) {
+                chunk_end += 1;
+            }
+        }
+
+        fragments.push(Fragment {
+            content: &code[current_start..chunk_end],
+            start_line: line_number,
+            end_line: line_number,
+            file_path: path,
+            start_byte_index: current_start.into(),
+            end_byte_index: chunk_end.into(),
+        });
+
+        current_start = chunk_end;
+    }
+
+    fragments
+}
+
+#[cfg(test)]
+#[path = "naive_tests.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/chunker/naive_tests.rs

@@ -0,0 +1,354 @@
+use super::*;
+use std::path::Path;
+
+#[test]
+fn test_chunker() {
+    let code = "This is some text content\nthat should be chunked\nusing the naive chunker\nbecause the language isn't recognized.";
+    let path = Path::new("test_file.xyz");
+
+    let max_lines = 1;
+    let fragments = chunk_code(code, path, 10000, max_lines);
+
+    assert!(!fragments.is_empty(), "Expected at least one fragment");
+
+    assert_eq!(fragments.len(), code.lines().count());
+    for (idx, line) in code.lines().enumerate() {
+        assert_eq!(fragments[idx].content, line);
+        assert_eq!(fragments[idx].start_line, idx);
+        assert_eq!(fragments[idx].end_line, idx);
+    }
+}
+
+#[test]
+fn test_chunker_large_chunk() {
+    let code = "This is some text content\nthat should be chunked\nusing the naive chunker\nbecause the language isn't recognized.";
+    let path = Path::new("test_file.xyz");
+
+    let fragments = chunk_code(code, path, 10000, 100);
+
+    // We should have only one fragment
+    assert_eq!(fragments.len(), 1);
+
+    assert_eq!(fragments[0].content, code);
+    assert_eq!(fragments[0].start_line, 0);
+    assert_eq!(fragments[0].end_line, code.lines().count() - 1);
+}
+
+#[test]
+fn test_chunker_max_bytes() {
+    // Create a string with known byte size - each line is exactly 20 bytes including newline
+    let code = "line1\nline2\nline3\nline4abcdefghijklmnopqrstuvwxyz";
+    let path = Path::new("test_file.xyz");
+
+    // Set max_bytes_per_chunk to 25 bytes to force multiple chunks for the last line (which is 30 bytes).
+    let max_bytes_per_chunk = 25;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Verify we have multiple chunks
+    assert!(
+        fragments.len() > 1,
+        "Expected multiple chunks due to size limit"
+    );
+
+    // Verify that no chunk exceeds the max_bytes_per_chunk limit
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            fragment.content.trim().len() <= max_bytes_per_chunk,
+            "Fragment {} has size {} bytes, which exceeds limit of {} bytes",
+            i,
+            fragment.content.len(),
+            max_bytes_per_chunk
+        );
+    }
+
+    // The first fragment contains all of the lines except the last one.
+    assert_eq!(fragments[0].content, "line1\nline2\nline3");
+
+    // The last two fragments contains the contents of the line line.
+    assert_eq!(fragments[1].content, "line4abcdefghijklmnopqrst");
+    assert_eq!(fragments[2].content, "uvwxyz");
+
+    // Verify that the chunks together contain all the original content
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    // Ignore any newlines when doing comparisons--the chunker may drop newlines at fragment boundaries
+    // and that's not necessary for testing the correctness of the naive chunker.
+    assert_eq!(
+        reassembled_content.replace('\n', ""),
+        code.replace('\n', ""),
+        "Reassembled content does not match original"
+    );
+}
+
+#[test]
+fn test_utf8_emoji_chunking() {
+    // Test with emojis (4-byte UTF-8 characters) to ensure byte boundaries are respected
+    let code = "Hello 🦀 Rust\nWorld 🌍 Test\n🚀 Rocket 🎯 Target";
+    let path = Path::new("test_emoji.txt");
+
+    // Set a small max_bytes_per_chunk to force splitting through emoji characters
+    let max_bytes_per_chunk = 15; // This will force splits in the middle of emoji sequences
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Verify we have multiple chunks
+    assert!(
+        fragments.len() > 1,
+        "Expected multiple chunks due to size limit"
+    );
+
+    // Verify that no chunk exceeds the max_bytes_per_chunk limit
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            fragment.content.len() <= max_bytes_per_chunk,
+            "Fragment {} has size {} bytes, which exceeds limit of {} bytes. Content: '{}'",
+            i,
+            fragment.content.len(),
+            max_bytes_per_chunk,
+            fragment.content
+        );
+    }
+
+    // Verify that all fragments contain valid UTF-8
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            fragment.content.is_ascii() || std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content
+        );
+    }
+
+    // Verify that reassembled content matches original (ignoring newlines)
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    assert_eq!(
+        reassembled_content.replace('\n', ""),
+        code.replace('\n', ""),
+        "Reassembled content does not match original"
+    );
+}
+
+#[test]
+fn test_utf8_accented_characters() {
+    // Test with accented characters (2-byte UTF-8)
+    let code = "Café résumé naïve\nÉlève découvrir\nMañana piñata";
+    let path = Path::new("test_accents.txt");
+
+    // Set max_bytes_per_chunk to force splitting through accented characters
+    let max_bytes_per_chunk = 10;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Verify we have multiple chunks
+    assert!(
+        fragments.len() > 1,
+        "Expected multiple chunks due to size limit"
+    );
+
+    // Verify that all fragments contain valid UTF-8
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content.as_bytes()
+        );
+    }
+
+    // Verify that reassembled content matches original (ignoring newlines)
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    assert_eq!(
+        reassembled_content.replace('\n', ""),
+        code.replace('\n', ""),
+        "Reassembled content does not match original"
+    );
+}
+
+#[test]
+fn test_utf8_mixed_characters() {
+    // Test with a mix of ASCII, 2-byte, 3-byte, and 4-byte UTF-8 characters
+    let code = "ASCII text 中文 🦀 résumé ℘ math symbols";
+    let path = Path::new("test_mixed.txt");
+
+    // Set a small chunk size to force many splits
+    let max_bytes_per_chunk = 8;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Verify we have multiple chunks
+    assert!(
+        fragments.len() > 1,
+        "Expected multiple chunks due to size limit"
+    );
+
+    // Verify that all fragments contain valid UTF-8 and don't exceed size limit
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            fragment.content.len() <= max_bytes_per_chunk,
+            "Fragment {} has size {} bytes, which exceeds limit of {} bytes",
+            i,
+            fragment.content.len(),
+            max_bytes_per_chunk
+        );
+
+        assert!(
+            std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content.as_bytes()
+        );
+    }
+
+    // Verify that reassembled content matches original
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    assert_eq!(
+        reassembled_content, code,
+        "Reassembled content does not match original"
+    );
+}
+
+#[test]
+fn test_utf8_boundary_edge_cases() {
+    // Test edge case where chunk boundary falls exactly on a multi-byte character
+    let code = "ab🦀cd"; // 'ab' (2 bytes) + '🦀' (4 bytes) + 'cd' (2 bytes) = 8 bytes total
+    let path = Path::new("test_edge.txt");
+
+    // Set chunk size to 3 bytes, which would split in the middle of the emoji without our fix
+    let max_bytes_per_chunk = 3;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Should have multiple fragments
+    assert!(fragments.len() >= 2, "Expected at least 2 fragments");
+
+    // Verify all fragments are valid UTF-8
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content.as_bytes()
+        );
+    }
+
+    // Verify reassembled content matches original
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    assert_eq!(
+        reassembled_content, code,
+        "Reassembled content does not match original"
+    );
+}
+
+#[test]
+fn test_utf8_single_multibyte_character() {
+    // Test with a single multi-byte character that's larger than chunk size
+    let code = "🦀"; // 4-byte emoji
+    let path = Path::new("test_single.txt");
+
+    // Set chunk size smaller than the character
+    let max_bytes_per_chunk = 2;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Should have exactly one fragment (can't split a single character)
+    assert_eq!(fragments.len(), 1, "Should have exactly one fragment");
+
+    // The fragment should contain the complete character
+    assert_eq!(fragments[0].content, code);
+
+    // Verify it's valid UTF-8
+    assert!(
+        std::str::from_utf8(fragments[0].content.as_bytes()).is_ok(),
+        "Fragment contains invalid UTF-8"
+    );
+}
+
+#[test]
+fn test_utf8_line_endings_with_multibyte() {
+    // Test multi-byte characters at line boundaries
+    let code = "Hello🌍\nWorld🦀\nTest🎯";
+    let path = Path::new("test_lines.txt");
+
+    let max_bytes_per_chunk = 10;
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1); // 1 line per chunk
+
+    // Should have 3 fragments (one per line)
+    assert_eq!(fragments.len(), 3, "Should have 3 fragments for 3 lines");
+
+    // Verify all fragments are valid UTF-8
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content.as_bytes()
+        );
+    }
+
+    // Verify line numbers are correct
+    assert_eq!(fragments[0].start_line, 0);
+    assert_eq!(fragments[0].end_line, 0);
+    assert_eq!(fragments[1].start_line, 1);
+    assert_eq!(fragments[1].end_line, 1);
+    assert_eq!(fragments[2].start_line, 2);
+    assert_eq!(fragments[2].end_line, 2);
+}
+
+#[test]
+fn test_panic_regression_byte_boundary() {
+    // This is a regression test for the "byte index is not a char boundary" panic.
+    // Before the fix, this would panic when trying to slice at byte index 3,
+    // which is in the middle of the 4-byte emoji '🦀'.
+    let code = "Hi🦀Test";
+    let path = Path::new("test_panic.txt");
+
+    // This chunk size would cause the original code to panic
+    let max_bytes_per_chunk = 3;
+
+    // This should not panic
+    let fragments = chunk_code(code, path, max_bytes_per_chunk, 1000);
+
+    // Verify we get valid fragments
+    assert!(!fragments.is_empty(), "Should have at least one fragment");
+
+    // Verify all fragments are valid UTF-8
+    for (i, fragment) in fragments.iter().enumerate() {
+        assert!(
+            std::str::from_utf8(fragment.content.as_bytes()).is_ok(),
+            "Fragment {} contains invalid UTF-8: {:?}",
+            i,
+            fragment.content.as_bytes()
+        );
+    }
+
+    // Verify reassembled content matches original
+    let reassembled_content: String = fragments
+        .iter()
+        .map(|f| f.content)
+        .collect::<Vec<_>>()
+        .join("");
+
+    assert_eq!(
+        reassembled_content, code,
+        "Reassembled content does not match original"
+    );
+}

crates/ai/src/index/full_source_code_embedding/chunker/semantic.rs

@@ -0,0 +1,170 @@
+use std::path::Path;
+
+use arborium::tree_sitter::{Language, Node, Parser, TreeCursor};
+use itertools::Itertools;
+
+use super::{coalesce_fragments, Fragment};
+
+/// Maximum depth for recursive tree traversal to prevent infinite recursion
+/// or excessive depth in malformed/deeply nested code.
+const MAX_TRAVERSAL_DEPTH: usize = 200;
+
+/// Chunks code into an ordered list of fragments, where each fragment is at most
+/// `max_bytes_per_chunk` bytes.
+pub(super) fn chunk_code<'a>(
+    code: &'a str,
+    path: &'a Path,
+    max_bytes_per_chunk: usize,
+    language: &Language,
+) -> anyhow::Result<Vec<Fragment<'a>>> {
+    // Wrap this in a block to ensure the treesitter Parser / Tree are dropped
+    // after creating the fragments.
+    let fragments = {
+        let mut parser = Parser::new();
+        parser.set_language(language)?;
+
+        let tree = parser
+            .parse(code, None /* old_tree */)
+            .ok_or_else(|| anyhow::anyhow!("Failed to parse code"))?;
+
+        let mut cursor = tree.walk();
+
+        let nodes = split_node(
+            tree.root_node(),
+            code,
+            max_bytes_per_chunk,
+            path,
+            &mut cursor,
+            0, // initial depth
+        )?;
+
+        coalesce_fragments(nodes.into_iter(), code, max_bytes_per_chunk)
+    };
+
+    // Release extra unused memory from malloc to the system.  For some
+    // reason, the memory obtained by the allocator is often not released
+    // back to the OS after we're done with it, resulting in high memory
+    // usage (from the perspective of the OS, though not from the perspective
+    // of the allocator).
+    //
+    // See: https://github.com/tree-sitter/tree-sitter/issues/3129
+    #[cfg(all(target_os = "linux", target_env = "gnu", not(feature = "jemalloc")))]
+    unsafe {
+        nix::libc::malloc_trim(0);
+    }
+
+    Ok(fragments)
+}
+
+/// Splits a [`Node`] into a series of [`Fragment`]s that are at most `max_bytes_per_chunk` bytes.
+fn split_node<'a, 'b>(
+    node: Node<'b>,
+    code: &'a str,
+    max_bytes_per_chunk: usize,
+    path: &'a Path,
+    cursor: &mut TreeCursor<'b>,
+    depth: usize,
+) -> anyhow::Result<Vec<Fragment<'a>>> {
+    // Check if we've exceeded the maximum traversal depth
+    if depth > MAX_TRAVERSAL_DEPTH {
+        return Err(anyhow::anyhow!(
+            "Maximum traversal depth {} exceeded, falling back to naive chunking",
+            MAX_TRAVERSAL_DEPTH
+        ));
+    }
+
+    let mut current_fragment = Fragment::from_node_start(node, path);
+    let mut fragments = vec![];
+
+    // Collect into a vec to avoid a double mutable borrow with `cursor` when we make
+    // the recursive call below.
+    for child in node.children(cursor).collect_vec() {
+        let child_size = child.end_byte().saturating_sub(child.start_byte());
+
+        // The child is larger than the max chunk size, so we need to split it recursively.
+        if child_size > max_bytes_per_chunk {
+            let mut new_fragment = Fragment::from_node_end(child, path);
+            std::mem::swap(&mut current_fragment, &mut new_fragment);
+            fragments.push(new_fragment);
+
+            fragments.append(&mut split_node(
+                child,
+                code,
+                max_bytes_per_chunk,
+                path,
+                cursor,
+                depth + 1,
+            )?);
+        } else if child_size + current_fragment.size() > max_bytes_per_chunk {
+            // The child would make the current fragment too large, so we finalize the current
+            // fragment and create a new one.
+            fragments.push(current_fragment);
+            current_fragment = Fragment::from_node_start(child, path);
+            current_fragment.append(&Fragment::from_node_end(child, path), code);
+        } else {
+            // The child fits within the current fragment.
+            current_fragment.end_line = child.end_position().row;
+            current_fragment.end_byte_index = child.end_byte().into();
+            current_fragment.content =
+                &code[current_fragment.start_byte_index.as_usize()..child.end_byte()];
+        }
+    }
+
+    fragments.push(current_fragment);
+
+    Ok(fragments)
+}
+
+impl<'a> Fragment<'a> {
+    /// Creates an empty fragment.
+    fn empty() -> Fragment<'a> {
+        Fragment {
+            content: "",
+            start_line: 0,
+            end_line: 0,
+            start_byte_index: 0.into(),
+            end_byte_index: 0.into(),
+            file_path: Path::new(""),
+        }
+    }
+
+    /// Creates a fragment comprised solely of the start of the given node.
+    fn from_node_start(node: Node<'_>, path: &'a Path) -> Self {
+        Fragment {
+            content: "",
+            start_line: node.start_position().row,
+            end_line: node.start_position().row,
+            start_byte_index: node.start_byte().into(),
+            end_byte_index: node.start_byte().into(),
+            file_path: path,
+        }
+    }
+
+    /// Creates a fragment comprised solely of the end of the given node.
+    fn from_node_end(node: Node<'_>, path: &'a Path) -> Self {
+        Fragment {
+            content: "",
+            start_line: node.end_position().row,
+            end_line: node.end_position().row,
+            start_byte_index: node.end_byte().into(),
+            end_byte_index: node.end_byte().into(),
+            file_path: path,
+        }
+    }
+
+    /// Creates a fragment comprised solely of the end of the given fragment.
+    fn from_fragment_end(fragment: &Fragment<'a>) -> Self {
+        Fragment {
+            content: "",
+            start_line: fragment.end_line,
+            end_line: fragment.end_line,
+            start_byte_index: fragment.end_byte_index,
+            end_byte_index: fragment.end_byte_index,
+            file_path: fragment.file_path,
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "semantic_tests.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/chunker/semantic_tests.rs

@@ -0,0 +1,90 @@
+use std::path::Path;
+
+use languages::language_by_filename;
+
+use super::*;
+
+#[test]
+fn test_basic_rust_chunking() {
+    let source_code = r#"
+#[derive(Debug)]
+struct Rectangle {
+    width: u32,
+    height: u32,
+}
+
+impl Rectangle {
+    fn area(&self) -> u32 {
+        self.width * self.height
+    }
+}
+
+fn main() {
+    let rect1 = Rectangle {
+        width: 30,
+        height: 50,
+    };
+
+    println!(
+        "The area of the rectangle is {} square pixels.",
+        rect1.area()
+    );
+}
+"#;
+
+    let max_chunk_size = 128;
+
+    let chunks = chunk_code(
+        source_code,
+        Path::new("test.rs"),
+        max_chunk_size,
+        &language_by_filename(Path::new("test.rs"))
+            .expect("Rust language must exist")
+            .grammar,
+    )
+    .unwrap();
+
+    assert_eq!(chunks.len(), 4);
+
+    // None of the chunks should exceed the chunk size.
+    for chunk in &chunks {
+        assert!(
+            chunk.content.len() <= max_chunk_size,
+            "Chunk should not exceed max size of {max_chunk_size} but was: {}",
+            chunk.content.len()
+        );
+    }
+
+    assert_eq!(
+        chunks[0].content.trim(),
+        r#"#[derive(Debug)]
+struct Rectangle {
+    width: u32,
+    height: u32,
+}"#
+    );
+    assert_eq!(
+        chunks[1].content.trim(),
+        r#"impl Rectangle {
+    fn area(&self) -> u32 {
+        self.width * self.height
+    }
+}"#
+    );
+    assert_eq!(
+        chunks[2].content.trim(),
+        r#"fn main() {
+    let rect1 = Rectangle {
+        width: 30,
+        height: 50,
+    };"#
+    );
+    assert_eq!(
+        chunks[3].content.trim(),
+        r#"println!(
+        "The area of the rectangle is {} square pixels.",
+        rect1.area()
+    );
+}"#
+    );
+}

crates/ai/src/index/full_source_code_embedding/fragment_metadata.rs

@@ -0,0 +1,148 @@
+use std::{collections::HashMap, ops::Range, path::PathBuf};
+
+use serde::{Deserialize, Serialize};
+use string_offset::ByteOffset;
+
+use super::merkle_tree::MerkleHash;
+use crate::index::full_source_code_embedding::chunker::Fragment;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct FragmentLocation {
+    pub start_line: usize,
+    /// End line number (inclusive).
+    pub end_line: usize,
+    /// The range of byte indices into the original source string for this fragment.
+    pub byte_range: Range<ByteOffset>,
+}
+
+/// Fragment metadata that we persist in the tree. This helps us map from a leaf merkle node
+/// to the actual content on user's disk.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct FragmentMetadata {
+    /// File path of the fragment.
+    pub absolute_path: PathBuf,
+    /// Location of the fragment within the file.
+    pub location: FragmentLocation,
+}
+
+impl FragmentMetadata {
+    /// Returns the estimated content size in bytes, derived from the stored byte range.
+    pub fn content_byte_size(&self) -> usize {
+        self.location
+            .byte_range
+            .end
+            .as_usize()
+            .saturating_sub(self.location.byte_range.start.as_usize())
+    }
+}
+
+impl From<&Fragment<'_>> for FragmentMetadata {
+    fn from(fragment: &Fragment<'_>) -> Self {
+        FragmentMetadata {
+            absolute_path: PathBuf::from(fragment.file_path),
+            location: FragmentLocation {
+                start_line: fragment.start_line,
+                end_line: fragment.end_line,
+                byte_range: fragment.start_byte_index..fragment.end_byte_index,
+            },
+        }
+    }
+}
+
+pub type LeafToFragmentMetadataMapping = HashMap<MerkleHash, Vec<FragmentMetadata>>;
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+pub struct LeafToFragmentMetadata {
+    mapping: LeafToFragmentMetadataMapping,
+}
+
+impl LeafToFragmentMetadata {
+    pub(super) fn empty() -> Self {
+        Self::default()
+    }
+
+    pub(super) fn new(initial_content: LeafToFragmentMetadataUpdates) -> Self {
+        Self::from(initial_content)
+    }
+
+    #[cfg(test)]
+    pub(super) fn new_for_test(content: HashMap<MerkleHash, Vec<FragmentMetadata>>) -> Self {
+        Self { mapping: content }
+    }
+
+    pub(super) fn mapping(&self) -> &LeafToFragmentMetadataMapping {
+        &self.mapping
+    }
+
+    pub fn get<T: AsRef<MerkleHash>>(&self, hash: T) -> Option<&Vec<FragmentMetadata>> {
+        self.mapping.get(hash.as_ref())
+    }
+
+    pub fn apply_update(&mut self, update: LeafToFragmentMetadataUpdates) {
+        let LeafToFragmentMetadataUpdates {
+            to_remove,
+            to_insert,
+        } = update;
+        for (path, hashes) in to_remove {
+            for hash in hashes {
+                let Some(mapping_entry) = self.mapping.get_mut(&hash) else {
+                    continue;
+                };
+                mapping_entry.retain(|metadata| metadata.absolute_path != path);
+                if mapping_entry.is_empty() {
+                    self.mapping.remove(&hash);
+                }
+            }
+        }
+        to_insert.into_iter().for_each(|(hash, metadatas)| {
+            self.mapping.entry(hash).or_default().extend(metadatas);
+        });
+    }
+}
+
+impl From<LeafToFragmentMetadataUpdates> for LeafToFragmentMetadata {
+    fn from(update: LeafToFragmentMetadataUpdates) -> Self {
+        Self {
+            mapping: update.to_insert,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct LeafToFragmentMetadataUpdates {
+    /// Since the same fragment can occur multiple times within the same file,
+    /// the filepath is not enough to uniquely identify a fragment.
+    /// At the moment, we only handle removing entire files, so using the path alone is okay.
+    /// In the future, add the FragmentMetadata to the key to uniquely identify a fragment.
+    pub(super) to_remove: HashMap<PathBuf, Vec<MerkleHash>>,
+    pub(super) to_insert: LeafToFragmentMetadataMapping,
+}
+
+impl LeafToFragmentMetadataUpdates {
+    pub fn empty() -> Self {
+        Self::default()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.to_remove.is_empty() && self.to_insert.is_empty()
+    }
+
+    pub fn merge(&mut self, other: Self) {
+        other.to_remove.into_iter().for_each(|(path, hashes)| {
+            self.to_remove.entry(path).or_default().extend(hashes);
+        });
+        other.to_insert.into_iter().for_each(|(hash, metadata)| {
+            self.to_insert.entry(hash).or_default().extend(metadata);
+        })
+    }
+
+    pub fn insertions(&self) -> &LeafToFragmentMetadataMapping {
+        &self.to_insert
+    }
+}
+
+impl Extend<LeafToFragmentMetadataUpdates> for LeafToFragmentMetadataUpdates {
+    fn extend<T: IntoIterator<Item = LeafToFragmentMetadataUpdates>>(&mut self, iter: T) {
+        iter.into_iter().for_each(|update| self.merge(update));
+    }
+}

crates/ai/src/index/full_source_code_embedding/merkle_tree/hash.rs

@@ -0,0 +1,227 @@
+//! Common types for hashes that identify codebase embedding state.
+
+use generic_array::GenericArray;
+use serde::{Deserialize, Serialize};
+use sha2::{digest::OutputSizeUser, Digest, Sha256};
+use std::{fmt, str::FromStr, sync::Arc};
+
+use crate::index::full_source_code_embedding::chunker::Fragment;
+
+use super::Error;
+
+/// The hash of an *intermediate* node in the [`MerkleTree`].
+///
+/// Unlike [`MerkleHash`], this is guaranteed to be an intermediate node.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct NodeHash(MerkleHash);
+
+impl NodeHash {
+    pub(super) fn new(hash: MerkleHash) -> Self {
+        Self(hash)
+    }
+}
+
+impl fmt::Display for NodeHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl FromStr for NodeHash {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(MerkleHash::from_str(s)?))
+    }
+}
+
+impl From<NodeHash> for warp_graphql::full_source_code_embedding::NodeHash {
+    fn from(value: NodeHash) -> Self {
+        warp_graphql::full_source_code_embedding::NodeHash(value.0.to_string())
+    }
+}
+
+impl TryFrom<warp_graphql::full_source_code_embedding::NodeHash> for NodeHash {
+    type Error = Error;
+
+    fn try_from(
+        value: warp_graphql::full_source_code_embedding::NodeHash,
+    ) -> Result<Self, Self::Error> {
+        Ok(Self(MerkleHash::from_str(&value.0)?))
+    }
+}
+
+impl AsRef<MerkleHash> for NodeHash {
+    fn as_ref(&self) -> &MerkleHash {
+        &self.0
+    }
+}
+
+impl From<&ContentHash> for NodeHash {
+    fn from(value: &ContentHash) -> Self {
+        NodeHash(value.0.to_owned())
+    }
+}
+
+impl From<ContentHash> for NodeHash {
+    fn from(value: ContentHash) -> Self {
+        NodeHash(value.0)
+    }
+}
+
+/// The hash of a fragment (leaf) node in the [`MerkleTree`].
+///
+/// Unlike [`MerkleHash`], this is guaranteed to be a leaf node.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct ContentHash(MerkleHash);
+
+impl ContentHash {
+    pub(crate) fn new(hash: MerkleHash) -> Self {
+        Self(hash)
+    }
+
+    pub fn from_content(content: &str) -> Self {
+        Self(MerkleHash::from_bytes(content.as_bytes()))
+    }
+}
+
+impl fmt::Display for ContentHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl FromStr for ContentHash {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(MerkleHash::from_str(s)?))
+    }
+}
+
+impl From<ContentHash> for warp_graphql::full_source_code_embedding::ContentHash {
+    fn from(value: ContentHash) -> Self {
+        warp_graphql::full_source_code_embedding::ContentHash(value.0.to_string())
+    }
+}
+
+impl TryFrom<warp_graphql::full_source_code_embedding::ContentHash> for ContentHash {
+    type Error = Error;
+
+    fn try_from(
+        value: warp_graphql::full_source_code_embedding::ContentHash,
+    ) -> Result<Self, Self::Error> {
+        Ok(Self(MerkleHash::from_str(&value.0)?))
+    }
+}
+
+impl AsRef<MerkleHash> for ContentHash {
+    fn as_ref(&self) -> &MerkleHash {
+        &self.0
+    }
+}
+
+impl AsRef<ContentHash> for ContentHash {
+    fn as_ref(&self) -> &ContentHash {
+        self
+    }
+}
+
+/// A SHA-256 hash for a node in the [`MerkleTree`].
+///
+/// Cloning a `MerkleHash` is cheap, and need not be avoided.
+/// TODO(CODE-399): make this private to the `merkle_tree` module.
+#[derive(Ord, PartialOrd, Eq, PartialEq, Hash, Clone)]
+pub(crate) struct MerkleHash(Arc<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>>);
+
+impl AsRef<MerkleHash> for MerkleHash {
+    fn as_ref(&self) -> &MerkleHash {
+        self
+    }
+}
+
+/// The default serialize prints the hash as a vector of small integers,
+/// which takes up more space (up to 5 characters per byte) and is more
+/// difficult to read.
+/// This custom serialization hex-encodes the bytes in a string instead.
+impl Serialize for MerkleHash {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut buf = [0u8; 64];
+        let hex_str = base16ct::lower::encode_str(&self.0, &mut buf)
+            .expect("Buffer is sufficient for a SHA-256 hash");
+        serializer.serialize_str(hex_str)
+    }
+}
+
+impl<'de> Deserialize<'de> for MerkleHash {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let hex_string = String::deserialize(deserializer)?;
+        MerkleHash::from_str(&hex_string).map_err(serde::de::Error::custom)
+    }
+}
+
+impl MerkleHash {
+    pub(super) fn from_hashes<'a>(iterator: impl Iterator<Item = &'a MerkleHash>) -> Self {
+        let mut hasher = Sha256::new();
+        for hash in iterator {
+            Digest::update(&mut hasher, hash.0.as_slice());
+        }
+
+        Self::from_digest(hasher)
+    }
+
+    pub(crate) fn from_bytes(content_bytes: &[u8]) -> Self {
+        let mut hasher = Sha256::new();
+        Digest::update(&mut hasher, content_bytes);
+
+        Self::from_digest(hasher)
+    }
+
+    pub(super) fn from_fragment(fragment: &Fragment<'_>) -> Self {
+        Self::from_bytes(fragment.content.as_bytes())
+    }
+
+    fn from_digest(digest: Sha256) -> Self {
+        Self(Arc::new(digest.finalize()))
+    }
+}
+
+impl FromStr for MerkleHash {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut buf: GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize> = Default::default();
+        let decoded =
+            base16ct::lower::decode(s.as_bytes(), &mut buf).map_err(Error::InvalidHash)?;
+        if decoded.len() != 32 {
+            return Err(Error::InvalidHash(base16ct::Error::InvalidLength));
+        }
+
+        Ok(MerkleHash(Arc::new(buf)))
+    }
+}
+
+impl fmt::Debug for MerkleHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "MerkleHash({self})")
+    }
+}
+
+impl fmt::Display for MerkleHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut buf = [0u8; 64];
+        let hex_string = base16ct::lower::encode_str(&self.0, &mut buf)
+            .expect("Buffer is sufficient for a SHA-256 hash");
+        write!(f, "{hex_string}")
+    }
+}
+
+#[cfg(test)]
+#[path = "hash_test.rs"]
+mod hash_test;

crates/ai/src/index/full_source_code_embedding/merkle_tree/hash_test.rs

@@ -0,0 +1,96 @@
+use super::MerkleHash;
+use crate::index::full_source_code_embedding::chunker::Fragment;
+use std::path::{Path, PathBuf};
+
+#[test]
+fn test_fragment_hash_from_content() {
+    let content = "fn main() { println!(\"Hello, world!\"); }";
+    let fragment = Fragment {
+        content,
+        file_path: Path::new("/foo/bar/bazz"),
+        start_line: 0,
+        end_line: 0,
+        start_byte_index: 0.into(),
+        end_byte_index: content.len().into(),
+    };
+    let hash = MerkleHash::from_fragment(&fragment);
+
+    assert_eq!(
+        "bb343b0950832ccd077f1515e842196f2ae4bb9e9261b0935ac57916c3cf305d",
+        hash.to_string()
+    );
+}
+
+#[test]
+fn test_node_hash_from_children() {
+    let path = PathBuf::from("/foo/bar/bazz");
+    let content1 = "fn func1() -> int { 1 }";
+    let content2 = "fn func2() -> int { 2 }";
+    let content3 = "fn func3() -> int { 3 }";
+
+    let leaf1 = MerkleHash::from_fragment(&Fragment {
+        content: content1,
+        file_path: path.as_path(),
+        start_line: 0,
+        end_line: 0,
+        start_byte_index: 0.into(),
+        end_byte_index: content1.len().into(),
+    });
+    let leaf2 = MerkleHash::from_fragment(&Fragment {
+        content: content2,
+        file_path: path.as_path(),
+        start_line: 1,
+        end_line: 1,
+        start_byte_index: content1.len().into(),
+        end_byte_index: (content1.len() + content2.len()).into(),
+    });
+    let leaf3 = MerkleHash::from_fragment(&Fragment {
+        content: content3,
+        file_path: path.as_path(),
+        start_line: 2,
+        end_line: 2,
+        start_byte_index: (content1.len() + content2.len()).into(),
+        end_byte_index: (content1.len() + content2.len() + content3.len()).into(),
+    });
+
+    // Create an iterator with the leaf hashes
+    let leaves = vec![&leaf1, &leaf2, &leaf3];
+    let hash = MerkleHash::from_hashes(leaves.into_iter());
+
+    assert_eq!(
+        "99c2f5b808870e4b1fcf163efbb588b6d5e074658d4ac43bab2eb5ffbe72c5cd",
+        hash.to_string()
+    );
+}
+
+#[test]
+fn test_merkle_hash_serialization_deserialization() {
+    // Create a MerkleHash from a fragment
+    let content = "fn main() { println!(\"Hello, world!\"); }";
+    let fragment = Fragment {
+        content,
+        file_path: Path::new("/foo/bar/bazz"),
+        start_line: 0,
+        end_line: 0,
+        start_byte_index: 0.into(),
+        end_byte_index: content.len().into(),
+    };
+    let original_hash = MerkleHash::from_fragment(&fragment);
+
+    // Serialize the hash to a JSON string
+    let serialized = serde_json::to_string(&original_hash).expect("Failed to serialize MerkleHash");
+
+    // Ensure serialized output is a hex string
+    assert!(serialized.starts_with("\""));
+    assert!(serialized.ends_with("\""));
+
+    // Deserialize back to a MerkleHash
+    let deserialized_hash: MerkleHash =
+        serde_json::from_str(&serialized).expect("Failed to deserialize MerkleHash");
+
+    // Verify deserialized hash matches the original
+    assert_eq!(original_hash, deserialized_hash);
+
+    // Also verify string representation matches
+    assert_eq!(original_hash.to_string(), deserialized_hash.to_string());
+}

crates/ai/src/index/full_source_code_embedding/merkle_tree/mod.rs

@@ -0,0 +1,32 @@
+use super::{chunker::Fragment, Error};
+
+mod hash;
+mod node;
+mod serialized_tree;
+mod tree;
+
+pub(super) use hash::MerkleHash;
+pub use hash::{ContentHash, NodeHash};
+pub(super) use node::NodeLens;
+pub(super) use serialized_tree::SerializedCodebaseIndex;
+pub(super) use tree::MerkleTree;
+
+use crate::index::Entry;
+cfg_if::cfg_if! {
+    if #[cfg(feature = "local_fs")] {
+        pub(super) use node::NodeId;
+        pub(super) use tree::TreeUpdateResult;
+    }
+}
+
+#[derive(Debug)]
+enum DirEntryOrFragment<'a> {
+    Entry(Entry),
+    Fragment(Fragment<'a>),
+}
+
+#[cfg(test)]
+mod test_util;
+
+#[cfg(test)]
+pub(super) use test_util::construct_test_merkle_tree;

crates/ai/src/index/full_source_code_embedding/merkle_tree/node.rs

@@ -0,0 +1,711 @@
+use crate::index::{
+    THREADPOOL, {DirectoryEntry, Entry, FileMetadata},
+};
+use anyhow::anyhow;
+
+use chrono::{DateTime, Utc};
+use itertools::Itertools;
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use repo_metadata::entry::is_file_parsable;
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use warp_util::standardized_path::StandardizedPath;
+
+use std::{
+    collections::{HashMap, HashSet},
+    ops::Range,
+    path::{Path, PathBuf},
+};
+use string_offset::ByteOffset;
+
+use crate::index::full_source_code_embedding::{
+    chunker::chunk_code,
+    fragment_metadata::{FragmentMetadata, LeafToFragmentMetadataUpdates},
+    Error,
+};
+
+use super::{
+    hash::MerkleHash,
+    serialized_tree::{SerializedFilesystemInfo, SerializedMerkleNode},
+    tree::UpdateFileResult,
+    ContentHash, DirEntryOrFragment, NodeHash,
+};
+
+/// ID that uniquely identifies a node in the merkle tree. It contains the node type
+/// as well as metadata that distinguishes nodes of the same type.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) enum NodeId {
+    /// A file node that contains fragment children
+    File {
+        absolute_path: PathBuf,
+        file_size: usize,
+        fs_modified_time: DateTime<Utc>,
+        file_contents_hash: String,
+    },
+    /// A directory node that contains file and directory children
+    Directory { absolute_path: PathBuf },
+    /// A leaf node representing a code fragment
+    Fragment {
+        absolute_path: PathBuf,
+        content_range: Range<ByteOffset>,
+    },
+}
+
+impl NodeId {
+    fn absolute_path(&self) -> &PathBuf {
+        match self {
+            Self::Directory { absolute_path } => absolute_path,
+            Self::File { absolute_path, .. } => absolute_path,
+            Self::Fragment { absolute_path, .. } => absolute_path,
+        }
+    }
+}
+
+/// A given node in the [`MerkleTree`].
+#[derive(Debug)]
+pub(super) struct MerkleNode {
+    /// The hash for the current node of the Merkle tree.
+    hash: MerkleHash,
+    /// The children of this merkle node.
+    children: Vec<MerkleNode>,
+    /// The ID of this node.
+    node_id: NodeId,
+}
+
+impl MerkleNode {
+    pub(super) fn new(
+        entry: DirEntryOrFragment<'_>,
+    ) -> Result<(MerkleNode, LeafToFragmentMetadataUpdates), Error> {
+        match entry {
+            DirEntryOrFragment::Entry(Entry::File(file)) => {
+                let local_path = file.path.to_local_path_lossy();
+                if !is_file_parsable(&local_path)? {
+                    return Err(Error::FileSizeExceeded);
+                }
+                let (file_size, fs_modified_time) = match std::fs::metadata(&local_path) {
+                    Ok(metadata) => {
+                        let file_size = metadata.len() as usize;
+                        if let Ok(fs_modified_time) = metadata.modified() {
+                            // Convert the SystemTime to DateTime<Utc>
+                            let fs_modified_time = fs_modified_time.into();
+
+                            (file_size, fs_modified_time)
+                        } else {
+                            log::warn!("Failed to get modified time for file {}", file.path);
+                            return Err(Error::FailedToGetMetadata(local_path));
+                        }
+                    }
+                    Err(_) => {
+                        log::warn!("Failed to get metadata for file {}", file.path);
+                        return Err(Error::FailedToGetMetadata(local_path));
+                    }
+                };
+
+                let file_contents = std::fs::read_to_string(&local_path)?;
+
+                // Compute SHA-256 hash of the file contents
+                let mut hasher = Sha256::new();
+                hasher.update(file_contents.as_bytes());
+                let file_contents_hash = format!("{:x}", hasher.finalize());
+
+                let fragments = chunk_code(&file_contents, &local_path);
+
+                let (children, mapping_updates): (Vec<_>, LeafToFragmentMetadataUpdates) =
+                    fragments
+                        .into_iter()
+                        .filter_map(|fragment| {
+                            Self::new(DirEntryOrFragment::Fragment(fragment)).ok()
+                        })
+                        .unzip();
+                if children.is_empty() {
+                    log::debug!(
+                        "Found empty file {} when generating the merkle tree",
+                        file.path
+                    );
+                    return Err(Error::EmptyNodeContent);
+                }
+
+                // Create a hash from all of the fragments of the file. We actively _do not_ sort here as the fragments
+                // are an ordered function of the file content.
+                let hash = MerkleHash::from_hashes(children.iter().map(|child| &child.hash));
+
+                // Add a small delay after file processing to reduce CPU spikes
+                // during large repository indexing, allowing other work to continue
+                std::thread::sleep(std::time::Duration::from_millis(5));
+
+                Ok((
+                    MerkleNode {
+                        hash,
+                        children,
+                        node_id: NodeId::File {
+                            absolute_path: file.path.to_local_path_lossy(),
+                            file_size,
+                            fs_modified_time,
+                            file_contents_hash,
+                        },
+                    },
+                    mapping_updates,
+                ))
+            }
+            DirEntryOrFragment::Entry(Entry::Directory(directory)) => {
+                let Some(pool) = THREADPOOL.as_ref() else {
+                    return Err(anyhow!("No threadpool exists for outline generation.").into());
+                };
+
+                let result = pool.install(|| {
+                    directory
+                        .children
+                        .into_par_iter()
+                        .filter_map(|node| Self::new(DirEntryOrFragment::Entry(node)).ok())
+                        .collect::<Vec<_>>()
+                });
+
+                let (mut children, mapping_updates): (Vec<_>, LeafToFragmentMetadataUpdates) =
+                    result.into_iter().unzip();
+
+                if children.is_empty() {
+                    return Err(Error::EmptyNodeContent);
+                }
+
+                // Sort the hashes to ensure we have consistent ordering for all files in the directory. We don't want a new
+                // hash if the `DirEntry`s are the same, but returned in a different order.
+                children.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
+
+                let hash = MerkleHash::from_hashes(children.iter().map(|child| &child.hash));
+                Ok((
+                    MerkleNode {
+                        hash,
+                        children,
+                        node_id: NodeId::Directory {
+                            absolute_path: directory.path.to_local_path_lossy(),
+                        },
+                    },
+                    mapping_updates,
+                ))
+            }
+            DirEntryOrFragment::Fragment(fragment) => {
+                if fragment.content.is_empty() {
+                    return Err(Error::EmptyNodeContent);
+                }
+                let hash = MerkleHash::from_fragment(&fragment);
+                let fragment_metadata = FragmentMetadata::from(&fragment);
+
+                let mut leaf_node_to_fragment_updates = LeafToFragmentMetadataUpdates::empty();
+                leaf_node_to_fragment_updates
+                    .to_insert
+                    .insert(hash.clone(), vec![fragment_metadata]);
+
+                Ok((
+                    MerkleNode {
+                        hash,
+                        children: vec![],
+                        node_id: NodeId::Fragment {
+                            absolute_path: fragment.file_path.to_path_buf(),
+                            content_range: fragment.start_byte_index..fragment.end_byte_index,
+                        },
+                    },
+                    leaf_node_to_fragment_updates,
+                ))
+            }
+        }
+    }
+
+    pub(super) fn from_serialized(
+        serialized_node: SerializedMerkleNode,
+        parent_path: &Path,
+    ) -> anyhow::Result<(MerkleNode, LeafToFragmentMetadataUpdates)> {
+        let hash = serialized_node.hash();
+
+        let mut children = vec![];
+        let mut leaf_node_to_fragment_updates = LeafToFragmentMetadataUpdates::empty();
+
+        let node_id = match serialized_node.fs_info {
+            SerializedFilesystemInfo::Directory { absolute_path } => {
+                NodeId::Directory { absolute_path }
+            }
+            SerializedFilesystemInfo::File {
+                absolute_path,
+                file_size,
+                fs_modified_time,
+                file_contents_hash,
+            } => NodeId::File {
+                absolute_path,
+                file_size,
+                fs_modified_time,
+                file_contents_hash,
+            },
+            SerializedFilesystemInfo::Fragment { location } => {
+                let file_path = parent_path.to_path_buf();
+                leaf_node_to_fragment_updates
+                    .to_insert
+                    .entry(hash.as_ref().clone())
+                    .or_default()
+                    .push(FragmentMetadata {
+                        absolute_path: file_path.clone(),
+                        location: (&location).into(),
+                    });
+
+                NodeId::Fragment {
+                    absolute_path: file_path.clone(),
+                    content_range: location.byte_range,
+                }
+            }
+        };
+
+        let absolute_path = node_id.absolute_path();
+
+        for child in serialized_node.children {
+            let (child_node, new_fragments) = Self::from_serialized(child, absolute_path)?;
+            leaf_node_to_fragment_updates.merge(new_fragments);
+            children.push(child_node);
+        }
+
+        Ok((
+            MerkleNode {
+                hash: hash.as_ref().clone(),
+                children,
+                node_id,
+            },
+            leaf_node_to_fragment_updates,
+        ))
+    }
+
+    // Recompute the hash for a given node. Note that for directories we expect the children to be sorted beforehand.
+    fn recompute_hash(&mut self) {
+        match &self.node_id {
+            NodeId::Directory { .. } | NodeId::File { .. } => {
+                self.hash = MerkleHash::from_hashes(self.children.iter().map(|child| &child.hash));
+            }
+            NodeId::Fragment { .. } => {
+                log::error!("Shouldn't need to recompute hash for fragments");
+            }
+        }
+    }
+
+    /// Remove a set of target file paths from this MerkleNode.
+    pub(super) fn remove_files(
+        &mut self,
+        paths: &mut HashSet<PathBuf>,
+        node_masks: &mut NodeMask,
+        node_to_fragment_updates: &mut LeafToFragmentMetadataUpdates,
+    ) -> UpdateFileResult {
+        match &self.node_id {
+            // Only visit a directory if it is the ancestor of the target path.
+            NodeId::Directory { absolute_path } => {
+                let mut paths_under_directory = filter_paths_under_directory(paths, absolute_path);
+
+                if paths_under_directory.is_empty() {
+                    return UpdateFileResult::NoChange;
+                }
+
+                // Track the indices that need to be removed.
+                let mut removal_idx = vec![];
+                let mut updated_idx = HashMap::new();
+                for (i, child) in self.children.iter_mut().enumerate() {
+                    let mut node = NodeMask::new(i);
+                    match child.remove_files(
+                        &mut paths_under_directory,
+                        &mut node,
+                        node_to_fragment_updates,
+                    ) {
+                        // If the node should be removed, remove it from the children.
+                        UpdateFileResult::Deleted => {
+                            removal_idx.push(i);
+                        }
+                        // If the node is updated, push the current index to node paths.
+                        UpdateFileResult::Updated => {
+                            updated_idx.insert(i - removal_idx.len(), node);
+                        }
+                        UpdateFileResult::NoChange => (),
+                    };
+
+                    if paths_under_directory.is_empty() {
+                        break;
+                    }
+                }
+
+                for idx in removal_idx.into_iter().rev() {
+                    self.children.remove(idx);
+                }
+
+                // If there is no more children to this merkle node, delete the current node.
+                if self.children.is_empty() {
+                    return UpdateFileResult::Deleted;
+                }
+
+                // We need to rebuild the children since the order of the children must be strictly sorted by their
+                // merkle hash.
+                let new_children_with_idx = self
+                    .children
+                    .drain(..)
+                    .enumerate()
+                    .sorted_by(|(_, a), (_, b)| a.hash.cmp(&b.hash));
+
+                for (new_idx, (old_idx, child)) in new_children_with_idx.enumerate() {
+                    if let Some(mut node) = updated_idx.remove(&old_idx) {
+                        node.index = new_idx;
+                        node_masks.add_child(node);
+                    }
+                    self.children.push(child);
+                }
+
+                if !updated_idx.is_empty() {
+                    log::error!("Updated index should be empty after an upsert request!");
+                }
+
+                // Otherwise, update the cache.
+                self.recompute_hash();
+                UpdateFileResult::Updated
+            }
+            // Only visit a file if it matches the target path.
+            NodeId::File { absolute_path, .. } if paths.remove(absolute_path) => {
+                node_to_fragment_updates.to_remove.insert(
+                    absolute_path.to_path_buf(),
+                    self.child_hashes().cloned().collect_vec(),
+                );
+                UpdateFileResult::Deleted
+            }
+            _ => UpdateFileResult::NoChange,
+        }
+    }
+
+    /// Update / insert a batch target file paths to this MerkleNode. Return whether the current node is updated.
+    pub(super) fn upsert_files(
+        &mut self,
+        paths: &mut HashSet<PathBuf>,
+        node_masks: &mut NodeMask,
+        leaf_node_to_fragment_updates: &mut LeafToFragmentMetadataUpdates,
+    ) -> UpdateFileResult {
+        match &self.node_id {
+            NodeId::Directory { absolute_path } => {
+                let mut paths_under_directory = filter_paths_under_directory(paths, absolute_path);
+
+                if paths_under_directory.is_empty() {
+                    return UpdateFileResult::NoChange;
+                }
+
+                let mut updated_idx = HashMap::new();
+                let mut removal_idx = vec![];
+
+                for (i, child) in self.children.iter_mut().enumerate() {
+                    let mut node = NodeMask::new(i);
+                    let updated = child.upsert_files(
+                        &mut paths_under_directory,
+                        &mut node,
+                        leaf_node_to_fragment_updates,
+                    );
+
+                    // Only update node_masks if the child is updated.
+                    match updated {
+                        UpdateFileResult::Deleted => removal_idx.push(i),
+                        UpdateFileResult::Updated => {
+                            updated_idx.insert(i - removal_idx.len(), node);
+                        }
+                        UpdateFileResult::NoChange => (),
+                    }
+
+                    // There is no more things to update. We could break early.
+                    if paths_under_directory.is_empty() {
+                        break;
+                    }
+                }
+
+                for idx in removal_idx.into_iter().rev() {
+                    self.children.remove(idx);
+                }
+
+                // If there is no more children to this merkle node and we are not creating new nodes, delete the current node.
+                if self.children.is_empty() && paths_under_directory.is_empty() {
+                    return UpdateFileResult::Deleted;
+                }
+
+                let mut entries_to_create = Vec::new();
+
+                // If none of the existing child could be updated these, we need to create new nodes.
+                // The algorithm works as below:
+                // 1) Convert the to-be-inserted paths into a chain of ancestors (e.g. a/b/c -> [a, a/b, a/b/c]).
+                // 2) Dedupe the ancestors using a mapping of path -> directory entry.
+                // 3) Attach each directory node to its parent node.
+                // 4) Add the root-level directory nodes to entries_to_create.
+                //
+                // Note that for file nodes, we add them directly to entries_to_create if it is on the root level and
+                // to its corresponding parent node otherwise.
+                let mut created_dirs: HashMap<PathBuf, DirectoryEntry> = HashMap::new();
+
+                // First pass: Create all directory entries with empty children lists
+                for path in paths_under_directory {
+                    // Skip upserting full or non-existent directories.
+                    if path.is_dir() || !path.exists() {
+                        continue;
+                    }
+
+                    // Get all parent directories that need to be created
+                    let mut ancestors = Vec::new();
+                    let mut current = path.clone();
+
+                    // Start from the path's parent and work up to but not including absolute_path
+                    while let Some(parent) = current.parent() {
+                        current = parent.to_path_buf();
+
+                        if current == *absolute_path {
+                            break;
+                        }
+
+                        ancestors.push(current.clone());
+                    }
+
+                    if current != *absolute_path {
+                        log::warn!("Path should match absolute path before None");
+                        continue;
+                    }
+
+                    // Process ancestors from deepest to shallowest (reverse order)
+                    ancestors.reverse();
+
+                    // Create intermediate directories if they don't exist yet
+                    for ancestor in ancestors {
+                        created_dirs
+                            .entry(ancestor.clone())
+                            .or_insert_with(|| DirectoryEntry {
+                                path: StandardizedPath::try_from_local(&ancestor)
+                                    .expect("ancestor paths are always absolute"),
+                                children: Vec::new(),
+                                ignored: false,
+                                loaded: false,
+                            });
+                    }
+
+                    // Add the file entry to its parent directory's children list
+                    if let Some(parent_path) = path.parent() {
+                        if parent_path != absolute_path {
+                            if let Some(parent_dir) = created_dirs.get_mut(parent_path) {
+                                parent_dir
+                                    .children
+                                    .push(Entry::File(FileMetadata::new(path, false)));
+                            }
+                        } else {
+                            entries_to_create.push(Entry::File(FileMetadata::new(path, false)));
+                        }
+                    }
+                }
+
+                // Second pass: Sort directories by depth (deepest first) and establish relationships from bottom up
+                let mut dir_paths: Vec<PathBuf> = created_dirs.keys().cloned().collect();
+                // Sort by component count in reverse order - deeper paths first
+                dir_paths.sort_by_key(|p| std::cmp::Reverse(p.components().count()));
+
+                for dir_path in dir_paths {
+                    if let Some(parent_path) = dir_path.parent() {
+                        if let Some(child_dir) = created_dirs.remove(&dir_path) {
+                            // Skip if parent is the root directory
+                            if parent_path != absolute_path {
+                                if let Some(parent_dir) = created_dirs.get_mut(parent_path) {
+                                    // Now we have the completed child directory with all its children
+                                    parent_dir.children.push(Entry::Directory(child_dir));
+                                }
+                            } else {
+                                entries_to_create.push(Entry::Directory(child_dir));
+                            }
+                        }
+                    }
+                }
+
+                for entry in entries_to_create {
+                    let res = MerkleNode::new(DirEntryOrFragment::Entry(entry));
+                    let (child, mapping_updates) = match res {
+                        Ok((child, mapping)) => (child, mapping),
+                        // When encountering a node construction error, instead of early returning and interrupting the rest of the update,
+                        // consider it a skippable error.
+                        // TODO: We should capture and log these errors in the telemetry.
+                        Err(e) => {
+                            log::debug!("Failed to create new node for update: {e:#}");
+                            continue;
+                        }
+                    };
+
+                    self.children.push(child);
+                    leaf_node_to_fragment_updates.merge(mapping_updates);
+                    updated_idx.insert(self.children.len() - 1, NodeMask::new(0));
+                }
+
+                // We need to rebuild the children since the order of the children must be strictly sorted by their
+                // merkle hash.
+                let new_children_with_idx = self
+                    .children
+                    .drain(..)
+                    .enumerate()
+                    .sorted_by(|(_, a), (_, b)| a.hash.cmp(&b.hash));
+
+                for (new_idx, (old_idx, child)) in new_children_with_idx.enumerate() {
+                    if let Some(mut node) = updated_idx.remove(&old_idx) {
+                        node.index = new_idx;
+                        node_masks.add_child(node);
+                    }
+                    self.children.push(child);
+                }
+
+                if !updated_idx.is_empty() {
+                    log::error!("Updated index should be empty after an upsert request!");
+                }
+                self.recompute_hash();
+                UpdateFileResult::Updated
+            }
+            // For files, only a single path can match at a single time.
+            NodeId::File { absolute_path, .. } if paths.remove(absolute_path) => {
+                leaf_node_to_fragment_updates.to_remove.insert(
+                    absolute_path.clone(),
+                    self.child_hashes().cloned().collect_vec(),
+                );
+
+                if !absolute_path.exists() {
+                    return UpdateFileResult::Deleted;
+                }
+
+                let (new_node, mapping_update) = match MerkleNode::new(DirEntryOrFragment::Entry(
+                    Entry::File(FileMetadata::new(absolute_path.clone(), false)),
+                )) {
+                    Ok(res) => res,
+                    // If we run into a file permission error / empty node / exceeded max file limit, delete the node since we can't
+                    // determine what's the updated content.
+                    Err(_) => return UpdateFileResult::Deleted,
+                };
+                leaf_node_to_fragment_updates.merge(mapping_update);
+                self.children = new_node.children;
+                self.hash = new_node.hash;
+                self.node_id = new_node.node_id;
+                UpdateFileResult::Updated
+            }
+            _ => UpdateFileResult::NoChange,
+        }
+    }
+
+    pub(super) fn absolute_path(&self) -> &Path {
+        self.node_id.absolute_path()
+    }
+
+    fn child_hashes(&self) -> impl Iterator<Item = &MerkleHash> {
+        self.children.iter().map(|child| &child.hash)
+    }
+
+    pub(super) fn children(&self) -> impl Iterator<Item = &MerkleNode> {
+        self.children.iter()
+    }
+
+    pub(super) fn count_children(&self) -> usize {
+        self.children.len()
+    }
+
+    pub(super) fn child_at(&self, index: usize) -> &MerkleNode {
+        &self.children[index]
+    }
+
+    pub(super) fn hash(&self) -> &MerkleHash {
+        &self.hash
+    }
+
+    pub(super) fn node_id(&self) -> &NodeId {
+        &self.node_id
+    }
+
+    pub(super) fn is_fragment(&self) -> bool {
+        matches!(self.node_id, NodeId::Fragment { .. })
+    }
+}
+
+fn filter_paths_under_directory(
+    paths: &mut HashSet<PathBuf>,
+    curr_path: &PathBuf,
+) -> HashSet<PathBuf> {
+    let mut paths_under_directory = HashSet::new();
+
+    // Construct and filter out paths that are under the current directory.
+    for path in paths.iter() {
+        if path.starts_with(curr_path) {
+            paths_under_directory.insert(path.clone());
+        }
+    }
+
+    for filter_path in paths_under_directory.iter() {
+        paths.remove(filter_path);
+    }
+
+    paths_under_directory
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct NodeLens<'a> {
+    node: &'a MerkleNode,
+}
+
+impl<'a> NodeLens<'a> {
+    pub(super) fn new(node: &'a MerkleNode) -> Self {
+        Self { node }
+    }
+
+    pub fn children(&self) -> impl Iterator<Item = NodeLens<'a>> {
+        self.node.children().map(|node| NodeLens { node })
+    }
+
+    pub fn hash(&self) -> NodeHash {
+        NodeHash::new(self.node.hash().clone())
+    }
+
+    pub fn content_hash(&self) -> Option<ContentHash> {
+        self.is_leaf()
+            .then(|| ContentHash::new(self.node.hash().clone()))
+    }
+
+    pub fn is_leaf(&self) -> bool {
+        self.node.is_fragment()
+    }
+
+    pub fn path(&self) -> &Path {
+        self.node.absolute_path()
+    }
+
+    pub(crate) fn node_id(&self) -> &NodeId {
+        self.node.node_id()
+    }
+}
+
+#[derive(Default)]
+pub(super) enum ChildrenPath {
+    #[default]
+    All,
+    SpecificChildren(Vec<NodeMask>),
+}
+
+/// A path from the leaf changed file node to the root.
+///
+/// Each NodeMask corresponds to a node in the tree. It contains the index of the node
+/// it is referencing in the parent node's children.
+///
+/// Note a NodeMask strictly couples with a snapshot of a Merkle tree. If the tree
+/// has been edited afterwards, the NodeMask will no longer be valid.
+#[derive(Default)]
+pub(super) struct NodeMask {
+    pub(super) index: usize,
+    pub(super) children: ChildrenPath,
+}
+
+impl NodeMask {
+    pub(super) fn new(index: usize) -> Self {
+        Self {
+            index,
+            children: Default::default(),
+        }
+    }
+
+    pub(super) fn add_child(&mut self, node: Self) {
+        match &mut self.children {
+            ChildrenPath::All => {
+                self.children = ChildrenPath::SpecificChildren(vec![node]);
+            }
+            ChildrenPath::SpecificChildren(children) => children.push(node),
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "node_test.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/merkle_tree/node_test.rs

@@ -0,0 +1,835 @@
+use crate::index::full_source_code_embedding::{
+    fragment_metadata::LeafToFragmentMetadataUpdates, merkle_tree::DirEntryOrFragment,
+};
+use repo_metadata::{DirectoryEntry, Entry};
+use virtual_fs::{Stub, VirtualFS};
+
+use std::collections::HashSet;
+
+use super::{MerkleNode, NodeMask};
+
+/// Tests that node hashes for directories are sorted (meaning they are resilient to files within
+/// the directory being in a different order).
+#[test]
+fn test_node_hash_for_directory_is_sorted() {
+    VirtualFS::test(
+        "test_node_hash_for_directory_is_sorted",
+        |dirs, mut sandbox| {
+            sandbox.with_files(vec![Stub::FileWithContent("foo", "foo")]);
+            sandbox.with_files(vec![Stub::FileWithContent("bar", "bar")]);
+
+            let mut directory_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                    .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: false,
+            };
+
+            for file in ["foo", "bar"] {
+                directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            let (node, _leaf_to_fragment_updates) = MerkleNode::new(DirEntryOrFragment::Entry(
+                Entry::Directory(directory_entry.clone()),
+            ))
+            .expect("Should be able to construct node");
+
+            directory_entry.children.clear();
+
+            for file in ["bar", "foo"] {
+                directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            let (node_reverse, _leaf_to_fragment_updates) =
+                MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(directory_entry)))
+                    .expect("Should be able to construct node");
+
+            // The node hashes should be the same even though the files were returned in different orders.
+            assert_eq!(node.hash, node_reverse.hash);
+        },
+    );
+}
+
+/// Tests that upserting a file updates the Merkle tree correctly.
+#[test]
+fn test_merkle_node_upsert_file() {
+    VirtualFS::test("test_merkle_node_upsert_file", |dirs, mut sandbox| {
+        // Create a directory with an initial file
+        sandbox.with_files(vec![Stub::FileWithContent(
+            "initial.txt",
+            "initial content",
+        )]);
+
+        let mut directory_entry = DirectoryEntry {
+            path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                .unwrap(),
+            children: vec![],
+            ignored: false,
+            loaded: true,
+        };
+
+        // Add the initial file to the directory entry
+        directory_entry
+            .find_or_insert_child(&dirs.tests().join("initial.txt"))
+            .expect("Should be able to insert into directory entry");
+
+        // Create the initial MerkleNode
+        let (mut node, initial_metadata_update) = MerkleNode::new(DirEntryOrFragment::Entry(
+            Entry::Directory(directory_entry.clone()),
+        ))
+        .expect("Should be able to construct node");
+
+        assert_eq!(
+            initial_metadata_update.to_insert.len(),
+            1,
+            "Should insert one file's metadata",
+        );
+        assert!(
+            initial_metadata_update.to_remove.is_empty(),
+            "Should not remove any file metadata",
+        );
+
+        let initial_root_hash = node.hash.clone();
+
+        // Create a new file to upsert
+        sandbox.with_files(vec![Stub::FileWithContent("new.txt", "new content")]);
+        directory_entry
+            .find_or_insert_child(&dirs.tests().join("new.txt"))
+            .expect("Should be able to insert into directory entry");
+
+        // Upsert the new file
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.upsert_files(
+            &mut HashSet::from([dirs.tests().join("new.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        // Verify the hash changed after adding a new file
+        assert_ne!(
+            initial_root_hash, node.hash,
+            "Hash should change after adding a new file",
+        );
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_insert.len(),
+            1,
+            "Upserting a new file should insert its content into metadata mapping",
+        );
+        assert!(
+            leaf_to_fragment_metadata_updates.to_remove.is_empty(),
+            "Inserting a new file should not remove any content from metadata mapping",
+        );
+
+        // Updated hash should be the same as reconstructing hash from scratch.
+        let hash_from_scratch = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+            directory_entry.clone(),
+        )))
+        .expect("Should be able to construct node")
+        .0
+        .hash;
+        assert_eq!(node.hash, hash_from_scratch);
+
+        // Remember the hash after adding the new file
+        let hash_after_add = node.hash.clone();
+
+        // Modify an existing file
+        sandbox.with_files(vec![Stub::FileWithContent(
+            "initial.txt",
+            "modified content",
+        )]);
+
+        // Upsert the modified file
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.upsert_files(
+            &mut HashSet::from([dirs.tests().join("initial.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        let updated_leaf_hash = leaf_to_fragment_metadata_updates
+            .to_insert
+            .keys()
+            .next()
+            .unwrap();
+        let updated_metadata_entry =
+            leaf_to_fragment_metadata_updates.to_insert[updated_leaf_hash].clone();
+
+        // Verify the hash changed after modifying a file
+        assert_ne!(
+            hash_after_add, node.hash,
+            "Hash should change after modifying a file"
+        );
+
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_insert.len(),
+            1,
+            "Upserting a modified file should insert its content into metadata mapping",
+        );
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_remove.len(),
+            1,
+            "Upserting a modified file should remove its old content from metadata mapping",
+        );
+
+        // Updated hash should be the same as reconstructing hash from scratch.
+        let hash_from_scratch = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+            directory_entry.clone(),
+        )))
+        .expect("Should be able to construct node")
+        .0
+        .hash;
+        assert_eq!(node.hash, hash_from_scratch);
+
+        // Remember the hash after modification
+        let hash_after_modify = node.hash.clone();
+
+        // Upsert with the same content (should not change the hash)
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.upsert_files(
+            &mut HashSet::from([dirs.tests().join("initial.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        // Verify the hash did not change when content is the same
+        assert_eq!(
+            hash_after_modify, node.hash,
+            "Hash should not change when upserting the same content"
+        );
+
+        // Verify that we remove and re-insert the file metadata in the leaf mapping.
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_remove.len(),
+            1,
+            "Should remove an entry from the metadata mapping",
+        );
+        assert!(
+            leaf_to_fragment_metadata_updates
+                .to_remove
+                .contains_key(&dirs.tests().join("initial.txt")),
+            "Upserting the same content should remove the old entry from the metadata mapping",
+        );
+
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_insert.len(),
+            1,
+            "Should re-insert an entry into the metadata mapping",
+        );
+        assert!(
+            leaf_to_fragment_metadata_updates
+                .to_insert
+                .contains_key(updated_leaf_hash),
+            "Upserting the same content re-inserts the same hash into the metadata mapping",
+        );
+        assert_eq!(
+            leaf_to_fragment_metadata_updates
+                .to_insert
+                .get(updated_leaf_hash)
+                .unwrap(),
+            &updated_metadata_entry,
+            "Upserting the same content re-inserts the same metadata into the metadata mapping"
+        );
+
+        // Test upserting a file in a new subdirectory that doesn't exist yet
+        let hash_before_subdirectory = node.hash.clone();
+
+        // Create a new subdirectory structure with a file
+        sandbox.mkdir("a");
+        sandbox.with_files(vec![Stub::FileWithContent(
+            "a/b.txt",
+            "subdirectory file content",
+        )]);
+
+        // Upsert the file in the new subdirectory
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.upsert_files(
+            &mut HashSet::from([dirs.tests().join("a/b.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        // Verify the hash changed after adding a file in a new subdirectory
+        assert_ne!(
+            hash_before_subdirectory, node.hash,
+            "Hash should change after adding a file in a new subdirectory"
+        );
+
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_insert.len(),
+            1,
+            "Upserting a file in a new subdirectory should insert its content into metadata mapping"
+        );
+        assert!(
+            leaf_to_fragment_metadata_updates.to_remove.is_empty(),
+            "Upserting a file in a new subdirectory should not remove any content from metadata mapping"
+        );
+
+        // Create a new MerkleTree with the expected structure manually
+        let mut expected_directory_entry = DirectoryEntry {
+            path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                .unwrap(),
+            children: vec![],
+            ignored: false,
+            loaded: true,
+        };
+
+        // Add both the existing files and new subdirectory structure
+        for file in ["initial.txt", "new.txt"] {
+            expected_directory_entry
+                .find_or_insert_child(&dirs.tests().join(file))
+                .expect("Should be able to insert into directory entry");
+        }
+
+        // Create a subdirectory entry for 'a'
+        let mut subdirectory_entry = DirectoryEntry {
+            path: warp_util::standardized_path::StandardizedPath::try_from_local(
+                &dirs.tests().join("a"),
+            )
+            .unwrap(),
+            children: vec![],
+            ignored: false,
+            loaded: true,
+        };
+
+        // Add the 'b.txt' file to the subdirectory
+        subdirectory_entry
+            .find_or_insert_child(&dirs.tests().join("a/b.txt"))
+            .expect("Should be able to insert into subdirectory entry");
+
+        // Add the subdirectory to the main directory
+        expected_directory_entry
+            .children
+            .push(Entry::Directory(subdirectory_entry));
+
+        // Create a MerkleNode with the expected structure
+        let (expected_node, _) = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+            expected_directory_entry,
+        )))
+        .expect("Should be able to construct node with expected structure");
+
+        // The hash of our modified node should match the hash of the manually constructed node
+        assert_eq!(
+            node.hash, expected_node.hash,
+            "Hash of node with upserted subdirectory should match hash of node constructed with expected structure"
+        );
+    });
+}
+
+/// Tests that removing a file updates the Merkle tree correctly.
+#[test]
+fn test_merkle_node_remove_file() {
+    VirtualFS::test("test_merkle_node_remove_file", |dirs, mut sandbox| {
+        // Create a directory with multiple files
+        sandbox.with_files(vec![
+            Stub::FileWithContent("file1.txt", "content 1"),
+            Stub::FileWithContent("file2.txt", "content 2"),
+            Stub::FileWithContent("file3.txt", "content 3"),
+        ]);
+
+        let mut directory_entry = DirectoryEntry {
+            path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                .unwrap(),
+            children: vec![],
+            ignored: false,
+            loaded: true,
+        };
+
+        // Add all files to the directory entry
+        for file in ["file1.txt", "file2.txt", "file3.txt"] {
+            directory_entry
+                .find_or_insert_child(&dirs.tests().join(file))
+                .expect("Should be able to insert into directory entry");
+        }
+
+        // Create the initial MerkleNode
+        let (mut node, initial_metadata_updates) = MerkleNode::new(DirEntryOrFragment::Entry(
+            Entry::Directory(directory_entry.clone()),
+        ))
+        .expect("Should be able to construct node");
+
+        assert_eq!(
+            initial_metadata_updates.to_insert.len(),
+            3,
+            "Should insert three files' metadata"
+        );
+        assert!(
+            initial_metadata_updates.to_remove.is_empty(),
+            "Should not remove any file metadata"
+        );
+
+        let initial_hash = node.hash.clone();
+
+        // Remove one of the files
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.remove_files(
+            &mut HashSet::from([dirs.tests().join("file2.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        // Verify the hash changed after removing a file
+        assert_ne!(
+            initial_hash, node.hash,
+            "Hash should change after removing a file"
+        );
+
+        // Verify that we update the fragment metadata
+        assert_eq!(
+            leaf_to_fragment_metadata_updates.to_remove.len(),
+            1,
+            "Removing a file should remove its content from metadata mapping"
+        );
+        assert!(
+            leaf_to_fragment_metadata_updates.to_insert.is_empty(),
+            "Removing a file should not insert any new metadata"
+        );
+
+        // Remember the hash after removal
+        let hash_after_remove = node.hash.clone();
+
+        // Try removing a non-existent file (should not change the hash)
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+        node.remove_files(
+            &mut HashSet::from([dirs.tests().join("nonexistent.txt")]),
+            &mut node_path,
+            &mut leaf_to_fragment_metadata_updates,
+        );
+
+        // Verify the hash did not change when removing a non-existent file
+        assert_eq!(
+            hash_after_remove, node.hash,
+            "Hash should not change when removing a non-existent file"
+        );
+
+        // Verify that there are no metadata updates
+        assert!(
+            leaf_to_fragment_metadata_updates.is_empty(),
+            "Metadata updates should be empty after trying to remove a non-existent file"
+        );
+
+        // Create a new MerkleNode with only the remaining files
+        let mut directory_entry_after_remove = DirectoryEntry {
+            path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                .unwrap(),
+            children: vec![],
+            ignored: false,
+            loaded: true,
+        };
+
+        for file in ["file1.txt", "file3.txt"] {
+            directory_entry_after_remove
+                .find_or_insert_child(&dirs.tests().join(file))
+                .expect("Should be able to insert into directory entry");
+        }
+
+        let (node_after_remove, _) = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+            directory_entry_after_remove,
+        )))
+        .expect("Should be able to construct node with remaining files");
+
+        // Verify that manually constructing a node without the removed file
+        // produces the same hash as removing the file from an existing node
+        assert_eq!(
+            node.hash, node_after_remove.hash,
+            "Hash after remove should match hash of node constructed without the file"
+        );
+    });
+}
+
+/// Tests that upserting and removing multiple files updates the Merkle tree correctly.
+#[test]
+fn test_merkle_node_multiple_operations() {
+    VirtualFS::test(
+        "test_merkle_node_multiple_operations",
+        |dirs, mut sandbox| {
+            // Create a directory with multiple initial files
+            sandbox.with_files(vec![
+                Stub::FileWithContent("file1.txt", "content 1"),
+                Stub::FileWithContent("file2.txt", "content 2"),
+                Stub::FileWithContent("file3.txt", "content 3"),
+            ]);
+
+            let mut directory_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                    .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: true,
+            };
+
+            // Add all initial files to the directory entry
+            for file in ["file1.txt", "file2.txt", "file3.txt"] {
+                directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            // Create the initial MerkleNode
+            let (mut node, initial_metadata_updates) = MerkleNode::new(DirEntryOrFragment::Entry(
+                Entry::Directory(directory_entry.clone()),
+            ))
+            .expect("Should be able to construct node");
+
+            // Ensure the initial leaf-node-to-fragment-metadata updates are correct.
+            assert_eq!(
+                initial_metadata_updates.to_insert.len(),
+                3,
+                "Should insert three files' metadata"
+            );
+
+            let initial_hash = node.hash.clone();
+
+            // Test 1: Upsert multiple files at once
+            // Create multiple new files to upsert
+            sandbox.with_files(vec![
+                Stub::FileWithContent("file4.txt", "content 4"),
+                Stub::FileWithContent("file5.txt", "content 5"),
+            ]);
+            for file in ["file4.txt", "file5.txt"] {
+                directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            // Upsert multiple files at once
+            let mut node_path = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.upsert_files(
+                &mut HashSet::from([
+                    dirs.tests().join("file4.txt"),
+                    dirs.tests().join("file5.txt"),
+                ]),
+                &mut node_path,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Verify the hash changed after adding multiple files
+            assert_ne!(
+                initial_hash, node.hash,
+                "Hash should change after adding multiple files"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_insert.len(),
+                2,
+                "Upserting new files should insert their content into metadata mapping"
+            );
+            assert!(
+                leaf_to_fragment_metadata_updates.to_remove.is_empty(),
+                "Upserting new files should not remove content from metadata mapping"
+            );
+
+            // Updated hash should be the same as reconstructing hash from scratch.
+            let hash_from_scratch = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+                directory_entry.clone(),
+            )))
+            .expect("Should be able to construct node")
+            .0
+            .hash;
+            assert_eq!(node.hash, hash_from_scratch);
+
+            let hash_after_multiple_add = node.hash.clone();
+
+            // Test 2: Remove multiple files at once
+            let mut node_path = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.remove_files(
+                &mut HashSet::from([
+                    dirs.tests().join("file1.txt"),
+                    dirs.tests().join("file3.txt"),
+                ]),
+                &mut node_path,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Verify the hash changed after removing multiple files
+            assert_ne!(
+                hash_after_multiple_add, node.hash,
+                "Hash should change after removing multiple files"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_remove.len(),
+                2,
+                "Removing multiple files should remove their content from metadata mapping"
+            );
+            assert!(
+                leaf_to_fragment_metadata_updates.to_insert.is_empty(),
+                "Removing multiple files should not insert any new metadata"
+            );
+
+            let mut directory_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                    .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: true,
+            };
+            for file in ["file2.txt", "file4.txt", "file5.txt"] {
+                directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            // Updated hash should be the same as reconstructing hash from scratch.
+            let hash_from_scratch = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+                directory_entry.clone(),
+            )))
+            .expect("Should be able to construct node")
+            .0
+            .hash;
+            assert_eq!(node.hash, hash_from_scratch);
+
+            let hash_after_multiple_remove = node.hash.clone();
+
+            // Test 3: Mixed operations - modify an existing file and add a new file
+            // Modify an existing file
+            sandbox.with_files(vec![
+                Stub::FileWithContent("file2.txt", "modified content 2"),
+                Stub::FileWithContent("file6.txt", "content 6"),
+            ]);
+            directory_entry
+                .find_or_insert_child(&dirs.tests().join("file6.txt"))
+                .expect("Should be able to insert into directory entry");
+
+            let mut node_path = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.upsert_files(
+                &mut HashSet::from([
+                    dirs.tests().join("file2.txt"),
+                    dirs.tests().join("file6.txt"),
+                ]),
+                &mut node_path,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Verify the hash changed after mixed operations
+            assert_ne!(
+                hash_after_multiple_remove, node.hash,
+                "Hash should change after mixed upsert operations"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_insert.len(),
+                2,
+                "Upserting modified and new files should insert their content into metadata mapping"
+            );
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_remove.len(),
+                1,
+                "Upserting modified and new files should remove the old content of the modified file from metadata mapping"
+            );
+
+            // Updated hash should be the same as reconstructing hash from scratch.
+            let hash_from_scratch = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+                directory_entry.clone(),
+            )))
+            .expect("Should be able to construct node")
+            .0
+            .hash;
+            assert_eq!(node.hash, hash_from_scratch);
+
+            let hash_after_mixed_upsert = node.hash.clone();
+
+            // Test 4: Edge case - upsert a file that already exists with the same content
+            let mut node_path = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.upsert_files(
+                &mut HashSet::from([dirs.tests().join("file2.txt")]),
+                &mut node_path,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Verify hash didn't change when upserting with the same content
+            assert_eq!(
+                hash_after_mixed_upsert, node.hash,
+                "Hash should not change when upserting files with the same content"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_remove.len(),
+                1,
+                "Upserting files with the same content first removes it from the metadata mapping"
+            );
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_insert.len(),
+                1,
+                "Upserting files with the same content re-adds it to the metadata mapping"
+            );
+
+            // Test 5: Edge case - remove a non-existent file while upserting a new file
+            // Create nested directories and a new file
+            sandbox.mkdir("subdir1");
+            sandbox.mkdir("subdir1/subdir2");
+            sandbox.with_files(vec![Stub::FileWithContent(
+                "subdir1/subdir2/nested.txt",
+                "nested content",
+            )]);
+
+            // Do mixed operations - remove non-existent file while upserting a new file in a subdirectory
+            let mut node_path_remove = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.remove_files(
+                &mut HashSet::from([dirs.tests().join("nonexistent.txt")]),
+                &mut node_path_remove,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Hash shouldn't change after trying to remove a non-existent file
+            assert_eq!(
+                hash_after_mixed_upsert, node.hash,
+                "Hash should not change when removing a non-existent file"
+            );
+
+            assert!(
+                leaf_to_fragment_metadata_updates.is_empty(),
+                "Removing a non-existent file should not modify metadata mapping"
+            );
+
+            // Add the nested file
+            let mut node_path_upsert = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.upsert_files(
+                &mut HashSet::from([dirs.tests().join("subdir1/subdir2/nested.txt")]),
+                &mut node_path_upsert,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Hash should change after adding the nested file
+            assert_ne!(
+                hash_after_mixed_upsert, node.hash,
+                "Hash should change after adding a file in a nested subdirectory"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_insert.len(),
+                1,
+                "Upserting a file in a nested subdirectory should insert its content into metadata mapping",
+            );
+            assert!(
+                leaf_to_fragment_metadata_updates.to_remove.is_empty(),
+                "Upserting a file in a nested subdirectory should not remove any files from metadata mapping",
+            );
+
+            let hash_after_nested_add = node.hash.clone();
+
+            // Test 6: Mixed operations - remove multiple files and add multiple files at once
+            sandbox.with_files(vec![
+                Stub::FileWithContent("new_file1.txt", "new content 1"),
+                Stub::FileWithContent("new_file2.txt", "new content 2"),
+            ]);
+
+            // Remove some files
+            let mut node_path_remove = NodeMask::default();
+            let mut leaf_to_fragment_metadata_updates = LeafToFragmentMetadataUpdates::empty();
+            node.remove_files(
+                &mut HashSet::from([
+                    dirs.tests().join("file4.txt"),
+                    dirs.tests().join("file5.txt"),
+                ]),
+                &mut node_path_remove,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Add new files
+            let mut node_path_upsert = NodeMask::default();
+            node.upsert_files(
+                &mut HashSet::from([
+                    dirs.tests().join("new_file1.txt"),
+                    dirs.tests().join("new_file2.txt"),
+                ]),
+                &mut node_path_upsert,
+                &mut leaf_to_fragment_metadata_updates,
+            );
+
+            // Verify the hash changed after these mixed operations
+            assert_ne!(
+                hash_after_nested_add, node.hash,
+                "Hash should change after removing and adding multiple files"
+            );
+
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_insert.len(),
+                2,
+                "Upserting multiple files should insert their content into metadata mapping"
+            );
+            assert_eq!(
+                leaf_to_fragment_metadata_updates.to_remove.len(),
+                2,
+                "Removing multiple files should remove their content from metadata mapping"
+            );
+
+            // Create a new MerkleTree with the expected final structure manually
+            let mut expected_directory_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests())
+                    .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: true,
+            };
+
+            // Add all files that should be in the final structure
+            for file in ["file2.txt", "file6.txt", "new_file1.txt", "new_file2.txt"] {
+                expected_directory_entry
+                    .find_or_insert_child(&dirs.tests().join(file))
+                    .expect("Should be able to insert into directory entry");
+            }
+
+            // Create nested subdirectory structure
+            let mut subdir1_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(
+                    &dirs.tests().join("subdir1"),
+                )
+                .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: true,
+            };
+
+            let mut subdir2_entry = DirectoryEntry {
+                path: warp_util::standardized_path::StandardizedPath::try_from_local(
+                    &dirs.tests().join("subdir1/subdir2"),
+                )
+                .unwrap(),
+                children: vec![],
+                ignored: false,
+                loaded: true,
+            };
+
+            // Add the nested file to its subdirectory
+            subdir2_entry
+                .find_or_insert_child(&dirs.tests().join("subdir1/subdir2/nested.txt"))
+                .expect("Should be able to insert into nested subdirectory entry");
+
+            // Build the directory hierarchy
+            subdir1_entry.children.push(Entry::Directory(subdir2_entry));
+
+            expected_directory_entry
+                .children
+                .push(Entry::Directory(subdir1_entry));
+
+            // Create a MerkleNode with the expected final structure
+            let (expected_node, _) = MerkleNode::new(DirEntryOrFragment::Entry(Entry::Directory(
+                expected_directory_entry,
+            )))
+            .expect("Should be able to construct node with expected structure");
+
+            // The hash of our modified node should match the hash of the manually constructed node
+            assert_eq!(
+            node.hash, expected_node.hash,
+            "Hash after all operations should match hash of node constructed with expected structure"
+        );
+        },
+    );
+}

crates/ai/src/index/full_source_code_embedding/merkle_tree/serialized_tree.rs

@@ -0,0 +1,200 @@
+use std::{
+    ops::Range,
+    path::{Path, PathBuf},
+};
+
+use super::{hash::MerkleHash, node::NodeId, MerkleTree, NodeHash, NodeLens};
+
+use crate::index::full_source_code_embedding::fragment_metadata::{
+    FragmentLocation, LeafToFragmentMetadata,
+};
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use string_offset::ByteOffset;
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub(crate) struct SerializedCodebaseIndex {
+    tree: SerializedMerkleTree,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub(crate) struct SerializedMerkleTree {
+    root: SerializedMerkleNode,
+}
+
+/// A given node in the [`MerkleTree`].
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub(super) struct SerializedMerkleNode {
+    /// The hash for the current node of the Merkle tree.
+    pub hash: MerkleHash,
+    /// The children of this merkle node.
+    pub children: Vec<SerializedMerkleNode>,
+    /// Node-specific details.
+    pub fs_info: SerializedFilesystemInfo,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub(super) struct SerializedFragmentLocation {
+    /// Start line number (inclusive).
+    pub start_line: usize,
+    /// End line number (inclusive).
+    pub end_line: usize,
+    /// The range of byte indices into the original source string for this fragment.
+    pub byte_range: Range<ByteOffset>,
+}
+
+impl From<&FragmentLocation> for SerializedFragmentLocation {
+    fn from(location: &FragmentLocation) -> Self {
+        Self {
+            start_line: location.start_line,
+            end_line: location.end_line,
+            byte_range: location.byte_range.clone(),
+        }
+    }
+}
+
+impl From<&SerializedFragmentLocation> for FragmentLocation {
+    fn from(serialized: &SerializedFragmentLocation) -> Self {
+        Self {
+            start_line: serialized.start_line,
+            end_line: serialized.end_line,
+            byte_range: serialized.byte_range.clone(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub(super) enum SerializedFilesystemInfo {
+    Directory {
+        absolute_path: PathBuf,
+    },
+    File {
+        /// The path of this node in the filesystem.
+        absolute_path: PathBuf,
+        /// File size in bytes.
+        file_size: usize,
+        /// Time the file was last modified, according to the filesystem.
+        fs_modified_time: DateTime<Utc>,
+        /// SHA-256 hash of the file contents.
+        file_contents_hash: String,
+    },
+    Fragment {
+        /// Fragment location within the file.
+        location: SerializedFragmentLocation,
+    },
+}
+
+impl SerializedCodebaseIndex {
+    pub fn new(
+        tree: &MerkleTree,
+        leaf_node_to_fragment_metadata: &LeafToFragmentMetadata,
+    ) -> anyhow::Result<Self> {
+        let root = SerializedMerkleNode::new(tree.root_node(), leaf_node_to_fragment_metadata)?;
+
+        Ok(Self {
+            tree: SerializedMerkleTree { root },
+        })
+    }
+
+    /// Consumes the index and returns just the tree.
+    pub(crate) fn into_tree(self) -> SerializedMerkleTree {
+        self.tree
+    }
+}
+
+impl SerializedMerkleTree {
+    pub(super) fn into_root(self) -> SerializedMerkleNode {
+        self.root
+    }
+}
+
+fn node_to_filesystem_info(
+    node: &NodeLens,
+    fragment_metadata_mapping: &LeafToFragmentMetadata,
+) -> anyhow::Result<SerializedFilesystemInfo> {
+    let absolute_path = node.path().to_path_buf();
+    match node.node_id() {
+        NodeId::Directory { .. } => Ok(SerializedFilesystemInfo::Directory { absolute_path }),
+        NodeId::File {
+            file_size,
+            fs_modified_time,
+            file_contents_hash,
+            ..
+        } => Ok(SerializedFilesystemInfo::File {
+            absolute_path,
+            file_size: *file_size,
+            fs_modified_time: *fs_modified_time,
+            file_contents_hash: file_contents_hash.clone(),
+        }),
+        NodeId::Fragment {
+            absolute_path,
+            content_range,
+        } => {
+            let Some(metadata_mapping) = fragment_metadata_mapping.get(node.hash()) else {
+                return Err(anyhow::anyhow!(
+                    "did not find hash in fragment metadata mapping"
+                ));
+            };
+
+            let Some(fragment) = metadata_mapping.iter().find(|fragment| {
+                fragment.absolute_path == *absolute_path
+                    && fragment.location.byte_range == *content_range
+            }) else {
+                return Err(anyhow::anyhow!(
+                    "did not find fragment metadata with matching path and content range"
+                ));
+            };
+
+            Ok(SerializedFilesystemInfo::Fragment {
+                location: (&fragment.location).into(),
+            })
+        }
+    }
+}
+
+impl SerializedMerkleNode {
+    fn new(
+        node: NodeLens,
+        fragment_metadata_mapping: &LeafToFragmentMetadata,
+    ) -> anyhow::Result<Self> {
+        let fs_info = node_to_filesystem_info(&node, fragment_metadata_mapping)?;
+
+        let children = node
+            .children()
+            .map(|child| Self::new(child, fragment_metadata_mapping))
+            .collect::<anyhow::Result<Vec<_>>>()?;
+        let hash = node.hash().as_ref().clone();
+        Ok(Self {
+            hash,
+            children,
+            fs_info,
+        })
+    }
+
+    /// Returns the node's absolute path if it is a file or directory.
+    /// Returns None if the node is a fragment.
+    pub(super) fn absolute_path(&self) -> Option<&Path> {
+        match &self.fs_info {
+            SerializedFilesystemInfo::Directory { absolute_path }
+            | SerializedFilesystemInfo::File { absolute_path, .. } => Some(absolute_path.as_path()),
+            SerializedFilesystemInfo::Fragment { .. } => None,
+        }
+    }
+
+    pub(super) fn hash(&self) -> NodeHash {
+        NodeHash::new(self.hash.to_owned())
+    }
+
+    pub(super) fn children(&self) -> impl Iterator<Item = &SerializedMerkleNode> {
+        self.children.iter()
+    }
+
+    pub(super) fn fs_info(&self) -> &SerializedFilesystemInfo {
+        &self.fs_info
+    }
+}
+
+#[cfg(test)]
+#[path = "serialized_tree_test.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/merkle_tree/serialized_tree_test.rs

@@ -0,0 +1,88 @@
+use futures::executor::block_on;
+use serde_json;
+use virtual_fs::VirtualFS;
+
+use crate::index::full_source_code_embedding::merkle_tree::{
+    construct_test_merkle_tree, MerkleTree,
+};
+
+use super::SerializedCodebaseIndex;
+
+#[test]
+fn round_trip_index_serialize_deserialize_json() {
+    VirtualFS::test("test_nodes_from_path_json", |dirs, mut sandbox| {
+        let (original_tree, original_metadata) =
+            block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+        let serializable_index = SerializedCodebaseIndex::new(&original_tree, &original_metadata);
+        let serializable_index =
+            serializable_index.expect("Should successfully construct serializable index");
+
+        let serialized_str =
+            serde_json::to_string(&serializable_index).expect("Should serialize to JSON string");
+        assert!(!serialized_str.is_empty());
+
+        let deserialized_index: SerializedCodebaseIndex =
+            serde_json::from_str(&serialized_str).expect("Should deserialize from JSON");
+        assert_eq!(
+            deserialized_index, serializable_index,
+            "Serialized struct should be identical"
+        );
+
+        let (reconstructed_tree, reconstructed_metadata) =
+            MerkleTree::from_serialized_tree(deserialized_index.into_tree())
+                .expect("Should rebuild Merkle Tree");
+        assert_eq!(
+            reconstructed_tree.root_node().hash(),
+            original_tree.root_node().hash(),
+            "Reconstructed Merkle tree should be identical",
+        );
+        assert_eq!(
+            original_metadata, reconstructed_metadata,
+            "Reconstructed metadata should be identical"
+        );
+    })
+}
+
+#[test]
+fn round_trip_index_serialize_deserialize_bincode() {
+    VirtualFS::test("test_nodes_from_path_bincode", |dirs, mut sandbox| {
+        let (original_tree, original_metadata) =
+            block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+        let serializable_index = SerializedCodebaseIndex::new(&original_tree, &original_metadata);
+        let serializable_index =
+            serializable_index.expect("Should successfully construct serializable index");
+
+        let serialized_bytes =
+            bincode::serialize(&serializable_index).expect("Should serialize to bincode");
+        assert!(!serialized_bytes.is_empty());
+
+        // Bincode output should be smaller than JSON
+        let json_bytes = serde_json::to_vec(&serializable_index).unwrap();
+        assert!(
+            serialized_bytes.len() < json_bytes.len(),
+            "Bincode ({} bytes) should be smaller than JSON ({} bytes)",
+            serialized_bytes.len(),
+            json_bytes.len(),
+        );
+
+        let deserialized_index: SerializedCodebaseIndex =
+            bincode::deserialize(&serialized_bytes).expect("Should deserialize from bincode");
+        assert_eq!(
+            deserialized_index, serializable_index,
+            "Serialized struct should be identical"
+        );
+
+        let (reconstructed_tree, reconstructed_metadata) =
+            MerkleTree::from_serialized_tree(deserialized_index.into_tree())
+                .expect("Should rebuild Merkle Tree");
+        assert_eq!(
+            reconstructed_tree.root_node().hash(),
+            original_tree.root_node().hash(),
+            "Reconstructed Merkle tree should be identical",
+        );
+        assert_eq!(
+            original_metadata, reconstructed_metadata,
+            "Reconstructed metadata should be identical"
+        );
+    })
+}

crates/ai/src/index/full_source_code_embedding/merkle_tree/test_util.rs

@@ -0,0 +1,99 @@
+use super::MerkleTree;
+use crate::index::full_source_code_embedding::fragment_metadata::LeafToFragmentMetadata;
+use repo_metadata::{DirectoryEntry, Entry};
+use virtual_fs::{Stub, VirtualFS};
+
+/// Construct a test Merkle tree with the following structure:
+/// ```
+/// root.txt
+/// top_dir/
+/// ├── file1.txt
+/// ├── subdir_a/
+/// │   ├── file2.txt
+/// │   └── file3.txt
+/// └── subdir_b/
+///     └── file4.txt
+/// ```
+#[cfg(test)]
+pub async fn construct_test_merkle_tree(
+    dirs: &virtual_fs::Dirs,
+    sandbox: &mut VirtualFS,
+) -> (MerkleTree, LeafToFragmentMetadata) {
+    sandbox.mkdir("top_dir");
+    sandbox.mkdir("top_dir/subdir_a");
+    sandbox.mkdir("top_dir/subdir_b");
+    sandbox.with_files(vec![
+        Stub::FileWithContent("root.txt", "root content"),
+        Stub::FileWithContent("top_dir/file1.txt", "file1 content"),
+        Stub::FileWithContent("top_dir/subdir_a/file2.txt", "file2 content"),
+        Stub::FileWithContent("top_dir/subdir_a/file3.txt", "file3 content"),
+        Stub::FileWithContent("top_dir/subdir_b/file4.txt", "file4 content"),
+    ]);
+
+    let mut root_dir_entry = DirectoryEntry {
+        path: warp_util::standardized_path::StandardizedPath::try_from_local(dirs.tests()).unwrap(),
+        children: vec![],
+        ignored: false,
+        loaded: true,
+    };
+    root_dir_entry
+        .find_or_insert_child(&dirs.tests().join("root.txt"))
+        .expect("Should be able to insert root file");
+
+    let mut top_dir_entry = DirectoryEntry {
+        path: warp_util::standardized_path::StandardizedPath::try_from_local(
+            &dirs.tests().join("top_dir"),
+        )
+        .unwrap(),
+        children: vec![],
+        ignored: false,
+        loaded: true,
+    };
+    top_dir_entry
+        .find_or_insert_child(&dirs.tests().join("top_dir/file1.txt"))
+        .expect("Should be able to insert file1");
+
+    let mut subdir_a_entry = DirectoryEntry {
+        path: warp_util::standardized_path::StandardizedPath::try_from_local(
+            &dirs.tests().join("top_dir/subdir_a"),
+        )
+        .unwrap(),
+        children: vec![],
+        ignored: false,
+        loaded: true,
+    };
+    subdir_a_entry
+        .find_or_insert_child(&dirs.tests().join("top_dir/subdir_a/file2.txt"))
+        .expect("Should be able to insert file2");
+    subdir_a_entry
+        .find_or_insert_child(&dirs.tests().join("top_dir/subdir_a/file3.txt"))
+        .expect("Should be able to insert file3");
+
+    let mut subdir_b_entry = DirectoryEntry {
+        path: warp_util::standardized_path::StandardizedPath::try_from_local(
+            &dirs.tests().join("top_dir/subdir_b"),
+        )
+        .unwrap(),
+        children: vec![],
+        ignored: false,
+        loaded: true,
+    };
+    subdir_b_entry
+        .find_or_insert_child(&dirs.tests().join("top_dir/subdir_b/file4.txt"))
+        .expect("Should be able to insert file4");
+
+    top_dir_entry
+        .children
+        .push(Entry::Directory(subdir_a_entry));
+    top_dir_entry
+        .children
+        .push(Entry::Directory(subdir_b_entry));
+
+    root_dir_entry
+        .children
+        .push(Entry::Directory(top_dir_entry));
+
+    MerkleTree::try_new(Entry::Directory(root_dir_entry))
+        .await
+        .expect("Should be able to construct tree")
+}

crates/ai/src/index/full_source_code_embedding/merkle_tree/tree.rs

@@ -0,0 +1,214 @@
+use crate::index::Entry;
+use anyhow::anyhow;
+use cfg_if::cfg_if;
+use std::{
+    collections::{HashSet, VecDeque},
+    path::PathBuf,
+};
+
+use crate::index::full_source_code_embedding::fragment_metadata::{
+    LeafToFragmentMetadata, LeafToFragmentMetadataUpdates,
+};
+use crate::index::full_source_code_embedding::Error;
+
+use super::{
+    node::{ChildrenPath, MerkleNode, NodeLens, NodeMask},
+    serialized_tree::SerializedMerkleTree,
+    DirEntryOrFragment,
+};
+
+pub(super) enum UpdateFileResult {
+    Deleted,
+    Updated,
+    NoChange,
+}
+
+pub(crate) struct TreeUpdateResult<'a> {
+    pub node_lens: Vec<NodeLens<'a>>,
+    pub leaf_to_fragment_meta_updates: LeafToFragmentMetadataUpdates,
+}
+
+/// A merkle tree used for codebase indexing. This data structure allows us to efficiently compute
+/// which parts of a repository have changed without needing to traverse every file in the tree.
+///
+/// The leaves of this tree are code fragments of a given file in the repository, with a
+/// corresponding SHA-256 hash of the contents.
+///
+/// The parents nodes in this tree are either a directory or a file (where the children are all the
+/// fragments of the file) and a corresponding hash of all the hashes of the children.
+///
+/// For example. Consider the following repository structure:
+/// * `/src`
+/// * `/src/foo.rs`
+/// * `/src/bar.rs`
+/// * `/src/bazz/buzz.rs`
+///
+/// The tree would roughly look:
+///
+/// /src (Hash: FooBarBuzzBazz)
+/// ├── /src/foo.rs (Hash: Foo)
+/// ├── /src/bar.rs (Hash: Bar)
+/// └── /src/bazz (Hash: BuzzBazz)
+///     └── /src/bazz/buzz.rs (Hash: Buzz)
+/// `
+#[derive(Debug)]
+pub(crate) struct MerkleTree {
+    root: MerkleNode,
+}
+
+impl MerkleTree {
+    /// Creates a new [`MerkleTree`] given the root node of an [`Entry`].
+    /// Returns an error if a node could not be created for any reason.
+    pub async fn try_new(entry: Entry) -> anyhow::Result<(MerkleTree, LeafToFragmentMetadata)> {
+        let build_node = move || MerkleNode::new(DirEntryOrFragment::Entry(entry));
+
+        let (root, mapping_update) = if tokio::runtime::Handle::try_current().is_ok() {
+            // Offload to a blocking thread so that the rayon `pool.install()` call inside
+            // `MerkleNode::new` does not block a tokio executor thread. Blocking executor
+            // threads starves other async tasks (e.g. shell history parsing during bootstrap).
+            tokio::task::spawn_blocking(build_node)
+                .await
+                .map_err(|e| anyhow::anyhow!("spawn_blocking join error: {e}"))?
+        } else {
+            build_node()
+        }?;
+
+        let leaf_node_to_fragment_metadata = LeafToFragmentMetadata::new(mapping_update);
+        Ok((Self { root }, leaf_node_to_fragment_metadata))
+    }
+
+    pub fn root_node(&self) -> NodeLens<'_> {
+        NodeLens::new(&self.root)
+    }
+
+    pub fn from_serialized_tree(
+        serialized_tree: SerializedMerkleTree,
+    ) -> anyhow::Result<(Self, LeafToFragmentMetadata)> {
+        let serialized_root = serialized_tree.into_root();
+        let Some(root_path) = serialized_root.absolute_path() else {
+            return Err(anyhow::anyhow!("root node should never be a fragment"));
+        };
+        let root_path = root_path.to_path_buf();
+        let (root, mapping_update) =
+            MerkleNode::from_serialized(serialized_root, root_path.as_path())?;
+        let leaf_node_to_fragment_metadata = LeafToFragmentMetadata::new(mapping_update);
+        Ok((Self { root }, leaf_node_to_fragment_metadata))
+    }
+
+    /// Construct the changed nodes' NodeLens from a NodeMask.
+    /// NodeLens are returned in reverse-BFS order (children first).
+    pub(super) fn nodes_from_mask(&self, node_mask: NodeMask) -> Result<Vec<NodeLens<'_>>, Error> {
+        let mut result = vec![];
+        let mut queue = VecDeque::new();
+        queue.push_back((&self.root, node_mask));
+
+        while let Some((current_node, path)) = queue.pop_front() {
+            result.push(NodeLens::new(current_node));
+
+            match path.children {
+                ChildrenPath::All => {
+                    for (idx, child_node) in current_node.children().enumerate() {
+                        queue.push_back((child_node, NodeMask::new(idx)));
+                    }
+                }
+                ChildrenPath::SpecificChildren(children_path) => {
+                    for child_path in children_path {
+                        if child_path.index < current_node.count_children() {
+                            let child_node = &current_node.child_at(child_path.index);
+                            queue.push_back((child_node, child_path));
+                        } else {
+                            return Err(Error::Other(anyhow!(
+                                "Invalid child index {} in node with {} child(ren)",
+                                child_path.index,
+                                current_node.count_children(),
+                            )));
+                        }
+                    }
+                }
+            }
+        }
+
+        result.reverse();
+        Ok(result)
+    }
+
+    /// Given the paths to a set of changed files, for each file, update the node if it exists in the tree, or
+    /// create a node if it doesn't exist. This also updates all the intermediate nodes.
+    ///
+    /// Return all the updated nodes.
+    pub async fn upsert_files(
+        &mut self,
+        mut paths: HashSet<PathBuf>,
+    ) -> Result<TreeUpdateResult<'_>, Error> {
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_meta_updates = LeafToFragmentMetadataUpdates::empty();
+
+        let mut do_upsert = || {
+            self.root.upsert_files(
+                &mut paths,
+                &mut node_path,
+                &mut leaf_to_fragment_meta_updates,
+            )
+        };
+
+        cfg_if! {
+            if #[cfg(not(target_family = "wasm"))] {
+                let upsert_result = if tokio::runtime::Handle::try_current().is_ok() {
+                    // `upsert_files` is expensive and can block a background thread for a while,
+                    // so use `block_in_place` to tell tokio to move any tasks enqueued for this
+                    // thread to another thread (so that they might be able to run on a different
+                    // thread).
+                    tokio::task::block_in_place(do_upsert)
+                } else {
+                    do_upsert()
+                };
+            } else {
+                let upsert_result = do_upsert();
+            }
+        }
+
+        // Note that we cannot early return directly here on error. We need to make sure leaf_node_to_fragment_metadatas
+        // is properly written so the tree remains valid.
+        let node_lens = if !matches!(upsert_result, UpdateFileResult::NoChange) {
+            self.nodes_from_mask(node_path)?
+        } else {
+            vec![self.root_node()]
+        };
+
+        Ok(TreeUpdateResult {
+            node_lens,
+            leaf_to_fragment_meta_updates,
+        })
+    }
+
+    /// Given the path to a removed file, remove the node if it exists in the tree.
+    ///
+    /// Return all the updated nodes.
+    pub async fn remove_files(
+        &mut self,
+        mut paths: HashSet<PathBuf>,
+    ) -> Result<TreeUpdateResult<'_>, Error> {
+        let mut node_path = NodeMask::default();
+        let mut leaf_to_fragment_meta_updates = LeafToFragmentMetadataUpdates::empty();
+
+        // Note that we cannot early return directly here on error. We need to make sure leaf_node_to_fragment_metadatas
+        // is properly written so the tree remains valid.
+        let node_lens = match self.root.remove_files(
+            &mut paths,
+            &mut node_path,
+            &mut leaf_to_fragment_meta_updates,
+        ) {
+            UpdateFileResult::NoChange => vec![self.root_node()],
+            _ => self.nodes_from_mask(node_path)?,
+        };
+
+        Ok(TreeUpdateResult {
+            node_lens,
+            leaf_to_fragment_meta_updates,
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "tree_test.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/merkle_tree/tree_test.rs

@@ -0,0 +1,121 @@
+use crate::index::full_source_code_embedding::merkle_tree::{
+    construct_test_merkle_tree, node::ChildrenPath,
+};
+use futures::executor::block_on;
+
+use virtual_fs::VirtualFS;
+
+use super::*;
+
+#[test]
+fn test_nodes_from_path() {
+    VirtualFS::test("test_nodes_from_path", |dirs, mut sandbox| {
+        let (tree, _metadata) = block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+
+        // Test: path with single child
+        let single_path = NodeMask {
+            index: 0,
+            children: ChildrenPath::SpecificChildren(vec![NodeMask::new(0)]),
+        };
+        let result = tree.nodes_from_mask(single_path).expect("Should not fail");
+        assert_eq!(
+            result.len(),
+            3,
+            "Should return three nodes for single-level path: root, child and its children"
+        );
+
+        // Test: path with multiple levels
+        // top_dir/file1.txt
+        let multi_path = NodeMask {
+            index: 0,
+            children: ChildrenPath::SpecificChildren(vec![NodeMask {
+                index: 1,
+                children: ChildrenPath::SpecificChildren(vec![NodeMask {
+                    index: 1,
+                    children: ChildrenPath::SpecificChildren(vec![NodeMask::new(0)]),
+                }]),
+            }]),
+        };
+        let result = tree.nodes_from_mask(multi_path).expect("Should not fail");
+        assert_eq!(
+            result.len(),
+            4,
+            "Should return 4 nodes for multi-level path: root, top_dir, file1.txt, and file1.txt's contents"
+        );
+        assert!(
+            result.first().unwrap().is_leaf(),
+            "First node should be file1.txt's contents"
+        );
+
+        // Test: invalid index should return error
+        let invalid_path = NodeMask {
+            index: 0,
+            children: ChildrenPath::SpecificChildren(vec![NodeMask::new(999)]),
+        };
+        assert!(
+            tree.nodes_from_mask(invalid_path).is_err(),
+            "Should return error for invalid index"
+        );
+
+        // Test: verify nodes are returned in reverse BFS order (children first)
+        let path = NodeMask {
+            index: 0,
+            children: ChildrenPath::SpecificChildren(vec![
+                NodeMask {
+                    // root.txt
+                    index: 0,
+                    children: ChildrenPath::SpecificChildren(vec![NodeMask::new(0)]),
+                },
+                NodeMask {
+                    // top_dir
+                    index: 1,
+                    children: ChildrenPath::SpecificChildren(vec![
+                        NodeMask {
+                            // subdir_b
+                            index: 0,
+                            children: ChildrenPath::SpecificChildren(vec![NodeMask {
+                                // file4.txt
+                                index: 0,
+                                children: ChildrenPath::SpecificChildren(vec![NodeMask::new(0)]),
+                            }]),
+                        },
+                        NodeMask::new(1), // file1.txt
+                        NodeMask {
+                            // subdir_a
+                            index: 2,
+                            children: ChildrenPath::SpecificChildren(vec![
+                                NodeMask {
+                                    // file2.txt
+                                    index: 0,
+                                    children: ChildrenPath::SpecificChildren(vec![NodeMask::new(
+                                        0,
+                                    )]),
+                                },
+                                NodeMask {
+                                    // file3.txt
+                                    index: 1,
+                                    children: ChildrenPath::SpecificChildren(vec![NodeMask::new(
+                                        0,
+                                    )]),
+                                },
+                            ]),
+                        },
+                    ]),
+                },
+            ]),
+        };
+
+        let result = tree.nodes_from_mask(path).expect("Should not fail");
+
+        // Children should come first
+        assert!(result[0].is_leaf(), "First node should be a leaf");
+        assert!(result[1].is_leaf(), "Second node should be a leaf");
+        assert!(result[2].is_leaf(), "Third node should be a leaf");
+
+        // Last node should be root
+        assert!(
+            !result.last().unwrap().is_leaf(),
+            "Last node should not be a leaf"
+        );
+    });
+}

crates/ai/src/index/full_source_code_embedding/mod.rs

@@ -0,0 +1,203 @@
+mod changed_files;
+mod chunker;
+mod codebase_index;
+mod fragment_metadata;
+pub mod manager;
+mod merkle_tree;
+mod priority_queue;
+mod snapshot;
+pub mod store_client;
+mod sync_client;
+
+use std::{ops::Range, path::PathBuf, time::Duration};
+pub use sync_client::SyncTask;
+
+pub use codebase_index::{CodebaseIndex, RetrievalID, SyncProgress};
+pub use merkle_tree::{ContentHash, NodeHash};
+
+use fragment_metadata::FragmentMetadata;
+use string_offset::ByteOffset;
+use thiserror::Error;
+use warp_graphql::queries::rerank_fragments::FragmentLocationInput;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("File I/O error {0:#}")]
+    Io(#[from] std::io::Error),
+    #[error("Not a git repository")]
+    NotAGitRepository,
+    #[error("Build tree error {0:#}")]
+    BuildTreeError(#[from] crate::index::BuildTreeError),
+    #[error("Unsupported platform")]
+    UnsupportedPlatform,
+    #[error("Invalid hash: {0:#}")]
+    InvalidHash(base16ct::Error),
+    #[error("Empty node content")]
+    EmptyNodeContent,
+    #[error("Failed to get metadata")]
+    FailedToGetMetadata(PathBuf),
+    #[error("File size exceeds maximum limit")]
+    FileSizeExceeded,
+    #[error(transparent)]
+    InconsistentState(#[from] InconsistentStateError),
+    #[error("Failed to generate embeddings for some hashes")]
+    FailedToGenerateEmbeddings(Vec<FragmentMetadata>),
+    #[error("Failed to sync some intermediate nodes")]
+    FailedToSyncIntermediateNodes(Vec<NodeHash>),
+    #[error("Diff merkle tree {0:#}")]
+    DiffMerkleTreeError(#[from] crate::index::full_source_code_embedding::DiffMerkleTreeError),
+    #[error("File system changed since merkle tree construction")]
+    FileSystemStateChanged,
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+    #[error("Failed to parse snapshot")]
+    SnapshotParsingFailed,
+}
+
+// Based off of BuildTreeError in entry.rs
+#[derive(Debug, Error)]
+pub enum DiffMerkleTreeError {
+    #[error("Merkle tree node and file mismatch")]
+    CurrentNodeMismatch(PathBuf),
+    #[error("File is ignored")]
+    Ignored,
+    #[error("Symlink is not supported")]
+    Symlink,
+    #[error("Fragment node in diffing process")]
+    Fragment(PathBuf),
+    #[error("Max depth exceeded")]
+    MaxDepthExceeded,
+    #[error("Exceeded max file limit")]
+    ExceededMaxFileLimit,
+}
+
+#[derive(Error, Debug)]
+pub enum InconsistentStateError {
+    #[error("Missing fragment metadata for {fragment_hash}")]
+    MissingFragmentMetadata { fragment_hash: ContentHash },
+    #[error("Can't find node index in merkle node")]
+    NodeIndexNotFound,
+}
+
+#[allow(non_camel_case_types)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub enum EmbeddingConfig {
+    OpenAiTextSmall3_256,
+    VoyageCode3_512,
+    Voyage3_5_Lite_512,
+    #[default]
+    Voyage3_5_512,
+}
+
+#[derive(Debug, Clone)]
+pub struct RepoMetadata {
+    pub path: Option<String>,
+}
+
+impl From<RepoMetadata> for warp_graphql::full_source_code_embedding::RepoMetadata {
+    fn from(val: RepoMetadata) -> Self {
+        Self { path: val.path }
+    }
+}
+
+impl From<EmbeddingConfig> for warp_graphql::full_source_code_embedding::EmbeddingConfig {
+    fn from(val: EmbeddingConfig) -> Self {
+        match val {
+            EmbeddingConfig::OpenAiTextSmall3_256 => {
+                warp_graphql::full_source_code_embedding::EmbeddingConfig::OpenaiTextSmall3256
+            }
+            EmbeddingConfig::VoyageCode3_512 => {
+                warp_graphql::full_source_code_embedding::EmbeddingConfig::VoyageCode3512
+            }
+            EmbeddingConfig::Voyage3_5_512 => {
+                warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35512
+            }
+            EmbeddingConfig::Voyage3_5_Lite_512 => {
+                warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35Lite512
+            }
+        }
+    }
+}
+
+impl TryFrom<warp_graphql::full_source_code_embedding::EmbeddingConfig> for EmbeddingConfig {
+    type Error = Error;
+
+    fn try_from(
+        value: warp_graphql::full_source_code_embedding::EmbeddingConfig,
+    ) -> Result<Self, Self::Error> {
+        match value {
+            warp_graphql::full_source_code_embedding::EmbeddingConfig::OpenaiTextSmall3256 => {
+                Ok(Self::OpenAiTextSmall3_256)
+            }
+            warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35Lite512 => {
+                Ok(Self::Voyage3_5_Lite_512)
+            }
+            warp_graphql::full_source_code_embedding::EmbeddingConfig::VoyageCode3512 => {
+                Ok(Self::VoyageCode3_512)
+            }
+            warp_graphql::full_source_code_embedding::EmbeddingConfig::Voyage35512 => {
+                Ok(Self::Voyage3_5_512)
+            }
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+pub struct CodebaseContextConfig {
+    pub embedding_config: EmbeddingConfig,
+    pub embedding_cadence: Duration,
+}
+
+#[derive(Clone)]
+pub struct FragmentLocation {
+    absolute_path: PathBuf,
+    byte_range: Range<ByteOffset>,
+}
+
+#[derive(Clone)]
+pub struct Fragment {
+    content: String,
+    content_hash: ContentHash,
+    location: FragmentLocation,
+}
+
+impl From<Fragment> for warp_graphql::full_source_code_embedding::Fragment {
+    fn from(val: Fragment) -> Self {
+        Self {
+            content: val.content,
+            content_hash: val.content_hash.into(),
+        }
+    }
+}
+
+impl From<Fragment> for warp_graphql::queries::rerank_fragments::RerankFragmentInput {
+    fn from(val: Fragment) -> Self {
+        Self {
+            content: val.content,
+            content_hash: val.content_hash.into(),
+            location: FragmentLocationInput {
+                byte_start: val.location.byte_range.start.as_usize() as i32,
+                byte_end: val.location.byte_range.end.as_usize() as i32,
+                file_path: val.location.absolute_path.to_string_lossy().to_string(),
+            },
+        }
+    }
+}
+
+impl TryFrom<warp_graphql::queries::rerank_fragments::RerankFragment> for Fragment {
+    type Error = Error;
+
+    fn try_from(
+        val: warp_graphql::queries::rerank_fragments::RerankFragment,
+    ) -> Result<Self, Self::Error> {
+        Ok(Self {
+            content: val.content,
+            content_hash: val.content_hash.try_into()?,
+            location: FragmentLocation {
+                absolute_path: PathBuf::from(val.location.file_path),
+                byte_range: ByteOffset::from(val.location.byte_start as usize)
+                    ..ByteOffset::from(val.location.byte_end as usize),
+            },
+        })
+    }
+}

crates/ai/src/index/full_source_code_embedding/priority_queue.rs

@@ -0,0 +1,79 @@
+use std::hash::Hash;
+use std::path::PathBuf;
+
+use itertools::Itertools;
+use priority_queue::PriorityQueue;
+
+use crate::workspace::WorkspaceMetadata;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub(super) enum Priority {
+    ActiveSession = 0,
+    OpenSession = 1,
+    PersistedSnapshot = 2,
+}
+
+#[derive(Debug, Clone)]
+struct QueueEntry {
+    metadata: WorkspaceMetadata,
+}
+
+impl Hash for QueueEntry {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.metadata.path.hash(state);
+    }
+}
+
+impl PartialEq for QueueEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.metadata.path == other.metadata.path
+    }
+}
+
+impl Eq for QueueEntry {}
+
+#[derive(Debug, Default)]
+pub(super) struct BuildQueue {
+    queue: PriorityQueue<QueueEntry, Priority>,
+}
+
+impl BuildQueue {
+    pub(super) fn empty() -> Self {
+        Self::default()
+    }
+
+    pub(super) fn queued_metadata(&self) -> impl IntoIterator<Item = WorkspaceMetadata> + use<'_> {
+        self.queue.iter().map(|(entry, _)| entry.metadata.clone())
+    }
+
+    pub(super) fn new_with_persisted(snapshots_to_load: Vec<WorkspaceMetadata>) -> Self {
+        let mut queue = PriorityQueue::new();
+        queue.extend(
+            snapshots_to_load
+                .into_iter()
+                .sorted_by(WorkspaceMetadata::most_recently_touched)
+                .map(|entry| (QueueEntry { metadata: entry }, Priority::PersistedSnapshot)),
+        );
+
+        Self { queue }
+    }
+
+    /// Pulls the next index root path to sync from the priority queue and returns it.
+    pub fn pick_next_sync(&mut self) -> Option<WorkspaceMetadata> {
+        self.queue.pop().map(|(entry, _priority)| entry.metadata)
+    }
+
+    /// Adjusts the priority of a path in the queue if it exists.
+    pub(super) fn update_path_priority(&mut self, root_path: PathBuf, priority: Priority) {
+        // Exemplar is only used to lookup the item in the queue with the Eq implemented above
+        // It will not overwrite the found item.
+        let exemplar = QueueEntry {
+            metadata: WorkspaceMetadata {
+                path: root_path,
+                ..Default::default()
+            },
+        };
+
+        self.queue.change_priority(&exemplar, priority);
+    }
+}

crates/ai/src/index/full_source_code_embedding/snapshot.rs

@@ -0,0 +1,230 @@
+use chrono::Utc;
+#[cfg(feature = "local_fs")]
+use repo_metadata::Repository;
+use std::{
+    collections::HashSet,
+    hash::{DefaultHasher, Hash, Hasher},
+    path::{Path, PathBuf},
+    time::Duration,
+};
+#[cfg(feature = "local_fs")]
+use warpui::ModelHandle;
+
+cfg_if::cfg_if! {
+    if #[cfg(feature = "local_fs")] {
+        use super::Error as CodebaseIndexError;
+        use std::sync::Arc;
+        use warpui::ModelContext;
+        use anyhow::Context;
+        use warp_core::safe_info;
+        use super::{store_client::StoreClient, CodebaseIndex, EmbeddingConfig};
+    }
+}
+
+use crate::workspace::WorkspaceMetadata;
+
+/// Number of days after which an index snapshot should be considered expired.
+pub(super) const REPO_SNAPSHOT_SHELF_LIFE_DAYS: u64 = 30;
+
+/// The maximum lifetime of an index snapshot file, after which it
+/// should be considered expired and deleted.
+const REPO_SNAPSHOT_SHELF_LIFE_DURATION: Duration =
+    Duration::from_secs(60 * 60 * 24 * REPO_SNAPSHOT_SHELF_LIFE_DAYS);
+
+/// Subdirectory inside the app's statedirectory that holds snapshot files.
+const REPO_SNAPSHOT_SUBDIR_NAME: &str = "codebase_index_snapshots";
+
+/// Splits a list of codebase indices into invalid and valid indices,
+/// based on their last write date and whether they have a corresponding snapshot file.
+pub(super) fn split_snapshot_metadata_by_validity(
+    persisted_codebase_indices: Vec<WorkspaceMetadata>,
+) -> (Vec<WorkspaceMetadata>, Vec<WorkspaceMetadata>) {
+    let now = Utc::now();
+    persisted_codebase_indices
+        .into_iter()
+        .partition(|index_metadata| {
+            log::info!(
+                "Discarding expired codebase index snapshot for {:?}",
+                index_metadata.path
+            );
+            index_metadata.is_expired(now, REPO_SNAPSHOT_SHELF_LIFE_DAYS)
+                || !has_snapshot(&index_metadata.path)
+        })
+}
+
+/// Delete snapshot files that are missing metadata or have expired from the snapshot directory.
+pub(super) fn clean_up_snapshot_files(
+    snapshot_file_dir: &Path,
+    persisted_codebase_indices: &[WorkspaceMetadata],
+) {
+    let expected_snapshot_filenames: HashSet<_> = persisted_codebase_indices
+        .iter()
+        .map(|index_metadata| snapshot_path(snapshot_file_dir, &index_metadata.path))
+        .collect();
+
+    if let Ok(fs_entries) = std::fs::read_dir(snapshot_file_dir) {
+        let fs_now = std::time::SystemTime::now();
+        for fs_entry in fs_entries.flatten() {
+            let path = fs_entry.path();
+
+            // Check if this is a regular file with a snapshot_ prefix
+            if let Ok(fs_metadata) = fs_entry.metadata() {
+                if fs_metadata.is_file() {
+                    maybe_clean_up_snapshot_file(
+                        &path,
+                        &fs_metadata,
+                        &fs_now,
+                        &expected_snapshot_filenames,
+                    );
+                }
+            }
+        }
+    }
+}
+
+fn maybe_clean_up_snapshot_file(
+    path: &Path,
+    fs_metadata: &std::fs::Metadata,
+    fs_now: &std::time::SystemTime,
+    expected_snapshot_filenames: &HashSet<PathBuf>,
+) {
+    if let Some(filename) = path.file_name() {
+        if filename.to_string_lossy().starts_with("snapshot_") {
+            let mut should_remove = false;
+
+            // Check if file itself is expired
+            if let Ok(modified_time) = fs_metadata.modified() {
+                if let Ok(age) = fs_now.duration_since(modified_time) {
+                    if age >= REPO_SNAPSHOT_SHELF_LIFE_DURATION {
+                        should_remove = true;
+                    }
+                }
+            }
+
+            // Check if file is not in alive_files set
+            if !expected_snapshot_filenames.contains(path) {
+                should_remove = true;
+            }
+
+            // Remove file if either condition is true
+            if should_remove {
+                if let Err(e) = std::fs::remove_file(path) {
+                    log::warn!("Failed to remove stale snapshot file {path:?}: {e}");
+                }
+            }
+        }
+    }
+}
+
+#[cfg(feature = "local_fs")]
+pub(super) fn read_snapshot(
+    store_client: Arc<dyn StoreClient>,
+    snapshot_dir: &Path,
+    repository: ModelHandle<Repository>,
+    max_files_repo_limit: usize,
+    embedding_generation_batch_size: usize,
+    ctx: &mut ModelContext<CodebaseIndex>,
+) -> anyhow::Result<CodebaseIndex> {
+    let repo_path_buf = repository.as_ref(ctx).root_dir().to_local_path_lossy();
+    let repo_path = repo_path_buf.as_path();
+    let snapshot_path = snapshot_path(snapshot_dir, repo_path);
+    let snapshot_bytes = std::fs::read(&snapshot_path)?;
+
+    let result = CodebaseIndex::new_from_snapshot(
+        repository,
+        store_client.clone(),
+        EmbeddingConfig::default(),
+        snapshot_bytes,
+        max_files_repo_limit,
+        embedding_generation_batch_size,
+        ctx,
+    );
+
+    // If rebuilding merkle tree from snapshot fails due to parsing error, delete the snapshot file.
+    if let Err(CodebaseIndexError::SnapshotParsingFailed) = result {
+        log::info!(
+            "Deleting invalid snapshot {:?} for repo",
+            snapshot_path.display(),
+        );
+        if let Err(e) = std::fs::remove_file(&snapshot_path) {
+            log::warn!(
+                "Failed to remove invalid snapshot file {:?}: {}",
+                snapshot_path.display(),
+                e
+            );
+        }
+    }
+
+    Ok(result?)
+}
+
+pub(super) fn has_snapshot(repo_path: &Path) -> bool {
+    let Some(snapshot_dir) = snapshot_dir() else {
+        return false;
+    };
+
+    let snapshot_path = snapshot_path(snapshot_dir.as_path(), repo_path);
+    snapshot_path.is_file()
+}
+
+/// Construct a directory to store index snapshots, if it doesn't already exist,
+/// and return its path.
+pub(super) fn snapshot_dir() -> Option<PathBuf> {
+    #[cfg(not(feature = "local_fs"))]
+    return None;
+
+    #[cfg(feature = "local_fs")]
+    {
+        let base_dir =
+            warp_core::paths::secure_state_dir().unwrap_or_else(warp_core::paths::state_dir);
+        let snapshot_dir_path = base_dir.join(REPO_SNAPSHOT_SUBDIR_NAME);
+
+        if !snapshot_dir_path.is_dir() {
+            std::fs::create_dir_all(&snapshot_dir_path).ok()?;
+        }
+        Some(snapshot_dir_path)
+    }
+}
+
+/// Constructs a snapshot path given a base directory and the codebase index's root path.
+pub(super) fn snapshot_path(snapshot_dir: &Path, repo_path: &Path) -> PathBuf {
+    // Use a hash the repo_path to create a unique filename
+    let mut hasher = DefaultHasher::new();
+    repo_path.hash(&mut hasher);
+    let snapshot_file_name = format!("snapshot_{}", hasher.finish());
+    snapshot_dir.join(snapshot_file_name)
+}
+
+#[cfg(test)]
+#[path = "snapshot_tests.rs"]
+mod tests;
+
+#[cfg(feature = "local_fs")]
+pub(super) fn migrate_snapshots_to_secure_dir_if_needed() -> anyhow::Result<()> {
+    // Only perform migration if a secure state directory is available.
+    let Some(secure_base) = warp_core::paths::secure_state_dir() else {
+        return Ok(());
+    };
+
+    let new_dir = secure_base.join(REPO_SNAPSHOT_SUBDIR_NAME);
+    let old_dir = warp_core::paths::state_dir().join(REPO_SNAPSHOT_SUBDIR_NAME);
+
+    if new_dir == old_dir {
+        return Ok(());
+    }
+
+    if old_dir.exists() && !new_dir.exists() {
+        if let Some(parent) = new_dir.parent() {
+            std::fs::create_dir_all(parent)
+                .context("Failed to create application data directory")?;
+        }
+        std::fs::rename(&old_dir, &new_dir)
+            .context("Failed to migrate codebase index snapshots")?;
+        safe_info!(
+            safe: ("Migrated codebase index snapshots into secure application container"),
+            full: ("Migrated codebase index snapshots from `{}` to `{}`", old_dir.display(), new_dir.display())
+        );
+    }
+
+    Ok(())
+}

crates/ai/src/index/full_source_code_embedding/snapshot_tests.rs

@@ -0,0 +1,107 @@
+use chrono::Duration;
+use virtual_fs::{Stub, VirtualFS};
+
+use super::*;
+
+#[test]
+fn test_clean_up_snapshot_files() {
+    VirtualFS::test("test_clean_up_snapshot_files", |dirs, mut sandbox| {
+        // Create snapshot directory with test files
+        sandbox.mkdir(REPO_SNAPSHOT_SUBDIR_NAME);
+
+        // Create a valid snapshot file
+        let test_path = PathBuf::from("/test/path");
+        let mut hasher = DefaultHasher::new();
+        test_path.hash(&mut hasher);
+        let valid_snapshot_name = format!("snapshot_{}", hasher.finish());
+
+        // Create entries in the virtual filesystem
+        let mut snapshot_dir_relative_path = PathBuf::new();
+        snapshot_dir_relative_path.push(REPO_SNAPSHOT_SUBDIR_NAME);
+        sandbox.with_files(vec![
+            // Valid snapshot file that matches metadata
+            Stub::FileWithContent(
+                snapshot_dir_relative_path
+                    .join(&valid_snapshot_name)
+                    .to_string_lossy()
+                    .as_ref(),
+                "valid content",
+            ),
+            // Expired snapshot file
+            Stub::FileWithContent(
+                snapshot_dir_relative_path
+                    .join("snapshot_expired")
+                    .to_string_lossy()
+                    .as_ref(),
+                "expired content",
+            ),
+            // Non-snapshot file that should be ignored
+            Stub::FileWithContent(
+                snapshot_dir_relative_path
+                    .join("regular_file.txt")
+                    .to_string_lossy()
+                    .as_ref(),
+                "regular content",
+            ),
+        ]);
+        // Subdirectory with the 'snapshot_' prefix that should be ignored
+        let invalid_subdir_path = snapshot_dir_relative_path.join("snapshot_prefixed_directory");
+        sandbox.mkdir(invalid_subdir_path.to_string_lossy().to_string().as_str());
+
+        let snapshot_dir_absolute_path = dirs.tests().join(REPO_SNAPSHOT_SUBDIR_NAME);
+
+        let valid_snapshot_file = snapshot_dir_absolute_path.join(&valid_snapshot_name);
+        let expired_file = snapshot_dir_absolute_path.join("snapshot_expired");
+        let regular_file = snapshot_dir_absolute_path.join("regular_file.txt");
+        let prefixed_subdir = snapshot_dir_absolute_path.join("snapshot_prefixed_directory");
+
+        assert!(valid_snapshot_file.is_file());
+        assert!(expired_file.is_file());
+        assert!(regular_file.is_file());
+        assert!(prefixed_subdir.is_dir());
+
+        // Set the expired file to be older than shelf life
+        let old_time = std::time::SystemTime::now()
+            - REPO_SNAPSHOT_SHELF_LIFE_DURATION
+            - Duration::days(1).to_std().unwrap();
+        filetime::set_file_mtime(
+            &expired_file,
+            filetime::FileTime::from_system_time(old_time),
+        )
+        .unwrap();
+
+        // Create test metadata that only includes the valid file
+        let metadata = vec![WorkspaceMetadata {
+            path: test_path,
+            navigated_ts: None,
+            modified_ts: None,
+            queried_ts: None,
+        }];
+
+        // Run cleanup
+        clean_up_snapshot_files(&snapshot_dir_absolute_path, &metadata);
+
+        // Valid snapshot should still exist
+        assert!(valid_snapshot_file.exists());
+
+        // Expired snapshot should be deleted
+        assert!(!expired_file.exists());
+
+        // Regular file should remain untouched
+        assert!(regular_file.exists());
+
+        // Subdirectory with 'snapshot_' prefix should remain untouched
+        assert!(prefixed_subdir.exists());
+    });
+}
+
+#[test]
+fn test_clean_up_snapshot_files_no_snapshot_dir() {
+    VirtualFS::test("test_clean_up_snapshot_files_no_dir", |dirs, _sandbox| {
+        // Test with empty metadata when snapshot directory doesn't exist
+        let snapshot_metadata = vec![];
+        let snapshot_directory = dirs.tests().join(REPO_SNAPSHOT_SUBDIR_NAME);
+        clean_up_snapshot_files(&snapshot_directory, &snapshot_metadata);
+        assert!(snapshot_metadata.is_empty());
+    });
+}

crates/ai/src/index/full_source_code_embedding/store_client.rs

@@ -0,0 +1,138 @@
+use async_trait::async_trait;
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    time::Duration,
+};
+
+use super::{
+    CodebaseContextConfig, ContentHash, EmbeddingConfig, Error, Fragment, NodeHash, RepoMetadata,
+};
+
+/// Client interface for a remote full source code embedding store.
+#[cfg_attr(not(target_family = "wasm"), async_trait::async_trait)]
+#[cfg_attr(target_family = "wasm", async_trait::async_trait(?Send))]
+pub trait StoreClient: 'static + Send + Sync {
+    /// Persist new intermediate Merkle tree nodes.
+    async fn update_intermediate_nodes(
+        &self,
+        embedding_config: EmbeddingConfig,
+        nodes: Vec<IntermediateNode>,
+    ) -> Result<HashMap<NodeHash, bool>, Error>;
+
+    /// Generate embeddings for individual code fragments.
+    ///
+    /// Embedding generation may fail on a per-fragment basis, so this returns the status of each
+    /// fragment. If the overall request fails, assume that no embeddings were generated.
+    async fn generate_embeddings(
+        &self,
+        embedding_config: EmbeddingConfig,
+        fragments: Vec<Fragment>,
+        root_hash: NodeHash,
+        repo_metadata: RepoMetadata,
+    ) -> Result<HashMap<ContentHash, bool>, Error>;
+
+    async fn populate_merkle_tree_cache(
+        &self,
+        embedding_config: EmbeddingConfig,
+        root_hash: NodeHash,
+        repo_metadata: RepoMetadata,
+    ) -> Result<bool, Error>;
+
+    async fn sync_merkle_tree(
+        &self,
+        nodes: Vec<NodeHash>,
+        embedding_config: EmbeddingConfig,
+    ) -> Result<HashSet<NodeHash>, Error>;
+
+    async fn rerank_fragments(
+        &self,
+        query: String,
+        fragment: Vec<Fragment>,
+    ) -> Result<Vec<Fragment>, Error>;
+
+    async fn get_relevant_fragments(
+        &self,
+        embedding_config: EmbeddingConfig,
+        query: String,
+        root_hash: NodeHash,
+        repo_metadata: RepoMetadata,
+    ) -> Result<Vec<ContentHash>, Error>;
+
+    async fn codebase_context_config(&self) -> Result<CodebaseContextConfig, Error>;
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct MockStoreClient;
+
+#[cfg_attr(not(target_family = "wasm"), async_trait)]
+#[cfg_attr(target_family = "wasm", async_trait(?Send))]
+impl StoreClient for MockStoreClient {
+    async fn update_intermediate_nodes(
+        &self,
+        _embedding_config: EmbeddingConfig,
+        _nodes: Vec<IntermediateNode>,
+    ) -> Result<HashMap<NodeHash, bool>, Error> {
+        Ok(HashMap::new())
+    }
+
+    async fn generate_embeddings(
+        &self,
+        _embedding_config: EmbeddingConfig,
+        _fragments: Vec<Fragment>,
+        _root_hash: NodeHash,
+        _repo_metadata: RepoMetadata,
+    ) -> Result<HashMap<ContentHash, bool>, Error> {
+        Ok(HashMap::new())
+    }
+
+    async fn sync_merkle_tree(
+        &self,
+        _nodes: Vec<NodeHash>,
+        _embedding_config: EmbeddingConfig,
+    ) -> Result<HashSet<NodeHash>, Error> {
+        Ok(HashSet::new())
+    }
+
+    async fn populate_merkle_tree_cache(
+        &self,
+        _embedding_config: EmbeddingConfig,
+        _root_hash: NodeHash,
+        _repo_metadata: RepoMetadata,
+    ) -> Result<bool, Error> {
+        Ok(true)
+    }
+
+    async fn rerank_fragments(
+        &self,
+        _query: String,
+        fragments: Vec<Fragment>,
+    ) -> Result<Vec<Fragment>, Error> {
+        // Return input as is for mock
+        Ok(fragments)
+    }
+
+    async fn get_relevant_fragments(
+        &self,
+        _embedding_config: EmbeddingConfig,
+        _query: String,
+        _root_hash: NodeHash,
+        _repo_metadata: RepoMetadata,
+    ) -> Result<Vec<ContentHash>, Error> {
+        Ok(Vec::new())
+    }
+
+    async fn codebase_context_config(&self) -> Result<CodebaseContextConfig, Error> {
+        Ok(CodebaseContextConfig {
+            embedding_config: EmbeddingConfig::default(),
+            embedding_cadence: Duration::from_secs(300),
+        })
+    }
+}
+
+/// The contents of an intermediate Merkle tree node, used to sync it to the remote embedding store.
+#[derive(Debug, Clone)]
+pub struct IntermediateNode {
+    pub hash: NodeHash,
+    pub children: Vec<NodeHash>,
+}

crates/ai/src/index/full_source_code_embedding/sync_client.rs

@@ -0,0 +1,604 @@
+use anyhow::{anyhow, Result};
+use itertools::Itertools;
+use std::future::Future;
+use std::ops::AddAssign;
+use std::pin::Pin;
+use std::{
+    collections::{HashMap, HashSet},
+    mem,
+    sync::Arc,
+};
+use warp_core::sync_queue::{IsTransientError, SyncQueue, SyncQueueTaskTrait};
+
+use super::{CodebaseContextConfig, NodeHash};
+
+use crate::index::full_source_code_embedding::store_client::IntermediateNode;
+
+use super::{
+    changed_files::ChangedFiles,
+    codebase_index::{build_fragments_from_metadata, SyncProgress},
+    fragment_metadata::LeafToFragmentMetadataMapping,
+    merkle_tree::{MerkleTree, NodeLens},
+    store_client::StoreClient,
+    EmbeddingConfig, Error, RepoMetadata,
+};
+use super::{ContentHash, Fragment};
+
+const SYNC_NODE_BATCH_SIZE: usize = 500;
+// Minimum node batch size used for updates.
+const MIN_UPDATE_NODE_BATCH_SIZE: usize = 100;
+
+/// Maximum total raw content bytes per `GenerateCodeEmbeddings` request.
+/// Set to 4 MB to stay under the 5 MB Cloud Armor limit after JSON serialization overhead.
+const MAX_BATCH_CONTENT_BYTES: usize = 4_000_000;
+
+#[derive(Debug, Clone, Default)]
+pub struct FlushFragmentResult {
+    pub fragment_count: usize,
+    pub total_fragment_size_bytes: usize,
+}
+
+impl AddAssign for FlushFragmentResult {
+    fn add_assign(&mut self, rhs: Self) {
+        self.fragment_count += rhs.fragment_count;
+        self.total_fragment_size_bytes += rhs.total_fragment_size_bytes;
+    }
+}
+
+pub struct GenerateEmbeddingsTask {
+    store_client: Arc<dyn StoreClient>,
+    embedding_config: EmbeddingConfig,
+    fragments: Vec<Fragment>,
+    root_node_hash: NodeHash,
+    repo_metadata: RepoMetadata,
+}
+
+pub struct UpdateIntermediateNodesTask {
+    store_client: Arc<dyn StoreClient>,
+    embedding_config: EmbeddingConfig,
+    nodes: Vec<IntermediateNode>,
+}
+
+pub struct SyncMerkleTreeTask {
+    store_client: Arc<dyn StoreClient>,
+    embedding_config: EmbeddingConfig,
+    nodes: Vec<NodeHash>,
+}
+
+pub enum SyncTask {
+    GenerateEmbeddings(GenerateEmbeddingsTask),
+    UpdateIntermediateNodes(UpdateIntermediateNodesTask),
+    SyncMerkleTree(SyncMerkleTreeTask),
+}
+
+#[derive(Debug)]
+pub enum SyncQueueResult {
+    GenerateEmbeddings(HashMap<ContentHash, bool>),
+    UpdateIntermediateNodes(HashMap<NodeHash, bool>),
+    SyncMerkleTree(HashSet<NodeHash>),
+}
+
+impl SyncQueueTaskTrait for SyncTask {
+    type Error = Error;
+    type Result = SyncQueueResult;
+    #[cfg(not(target_arch = "wasm32"))]
+    type Fut = Pin<Box<dyn Future<Output = Result<Self::Result, Self::Error>> + Send>>;
+    #[cfg(target_arch = "wasm32")]
+    type Fut = Pin<Box<dyn Future<Output = Result<Self::Result, Self::Error>>>>;
+    fn run(&mut self) -> Self::Fut {
+        match self {
+            SyncTask::GenerateEmbeddings(task) => {
+                let store_client = task.store_client.clone();
+                let embedding_config = task.embedding_config;
+                let fragments = task.fragments.clone();
+                let root_node_hash = task.root_node_hash.clone();
+                let repo_metadata = task.repo_metadata.clone();
+                Box::pin(async move {
+                    store_client
+                        .generate_embeddings(
+                            embedding_config,
+                            fragments,
+                            root_node_hash,
+                            repo_metadata,
+                        )
+                        .await
+                        .map(SyncQueueResult::GenerateEmbeddings)
+                })
+            }
+            SyncTask::SyncMerkleTree(task) => {
+                let store_client = task.store_client.clone();
+                let embedding_config = task.embedding_config;
+                let nodes = task.nodes.clone();
+                Box::pin(async move {
+                    store_client
+                        .sync_merkle_tree(nodes, embedding_config)
+                        .await
+                        .map(SyncQueueResult::SyncMerkleTree)
+                })
+            }
+            SyncTask::UpdateIntermediateNodes(task) => {
+                let store_client = task.store_client.clone();
+                let embedding_config = task.embedding_config;
+                let nodes = task.nodes.clone();
+                Box::pin(async move {
+                    store_client
+                        .update_intermediate_nodes(embedding_config, nodes)
+                        .await
+                        .map(SyncQueueResult::UpdateIntermediateNodes)
+                })
+            }
+        }
+    }
+}
+
+/// A sync client that is used to update the server merkle tree state so it is up-to-date with the client
+///
+/// A sync is broken down into two steps:
+/// 1) We need to walk the tree to find a list of nodes that need to be updated. This could be done by either
+///    a full scan of the tree or walking the tree from bottom up with a known changed leaf node.
+/// 2) With the list of nodes pending sync known, we could then generate embeddings for the leaf nodes and update
+///    intermediate nodes.
+pub(super) struct CodebaseIndexSyncOperation<'a> {
+    /// A list of dirty nodes that need to be synced with the server. Note that the nodes _MUST_ be ordered from children
+    /// to parent as we cannot sync parents that don't have their children synced.
+    nodes_pending_sync: Vec<NodeLens<'a>>,
+    store_client: Arc<dyn StoreClient>,
+    embedding_config: EmbeddingConfig,
+    sync_queue: SyncQueue<SyncTask>,
+    embedding_generation_batch_size: usize,
+}
+
+impl<'a> CodebaseIndexSyncOperation<'a> {
+    /// Perform a full sync of the merkle tree with the server. This guarantees we will add all inconsistent nodes
+    /// to the nodes_pending_sync list.
+    pub async fn full_sync(
+        tree: &'a MerkleTree,
+        store_client: Arc<dyn StoreClient>,
+        sync_queue: SyncQueue<SyncTask>,
+        sync_progress_tx: async_channel::Sender<SyncProgress>,
+        embedding_generation_batch_size: usize,
+    ) -> Result<(Self, CodebaseContextConfig), SyncOperationError> {
+        let config = store_client
+            .codebase_context_config()
+            .await
+            .map_err(SyncOperationError::ServerSyncError)?;
+
+        let mut operation = Self {
+            nodes_pending_sync: Vec::new(),
+            store_client,
+            embedding_config: config.embedding_config,
+            sync_queue,
+            embedding_generation_batch_size: embedding_generation_batch_size
+                .max(MIN_UPDATE_NODE_BATCH_SIZE),
+        };
+
+        let root_node = tree.root_node();
+        let mut nodes_pending_check = vec![root_node];
+
+        loop {
+            nodes_pending_check = operation
+                .check_if_nodes_synced(&nodes_pending_check, sync_progress_tx.clone())
+                .await?;
+
+            if nodes_pending_check.is_empty() {
+                break;
+            }
+        }
+
+        // We need to reverse the node orders here so we go from children -> parent;
+        operation.nodes_pending_sync.reverse();
+        Ok((operation, config))
+    }
+
+    /// Perform an incremental sync for a set of updated nodes.
+    /// This is used when we know exactly which nodes were modified through file system events.
+    ///
+    /// Returns the operation without flushing the nodes. The caller must call flush_nodes_pending_sync.
+    pub async fn incremental_sync(
+        updated_nodes: Vec<NodeLens<'a>>,
+        store_client: Arc<dyn StoreClient>,
+        sync_queue: SyncQueue<SyncTask>,
+        embedding_config: EmbeddingConfig,
+        sync_progress_tx: async_channel::Sender<SyncProgress>,
+        embedding_generation_batch_size: usize,
+    ) -> Result<Self, SyncOperationError> {
+        if updated_nodes.is_empty() {
+            log::info!("No nodes to sync incrementally");
+        } else {
+            log::debug!(
+                "Starting incremental sync preparation for {} updated nodes",
+                updated_nodes.len()
+            );
+        }
+
+        let mut operation = Self {
+            nodes_pending_sync: Vec::new(),
+            store_client,
+            embedding_config,
+            sync_queue,
+            embedding_generation_batch_size: embedding_generation_batch_size
+                .max(MIN_UPDATE_NODE_BATCH_SIZE),
+        };
+
+        // We need to check if nodes are synced in incremental sync since the to-be-update
+        // nodes could already exist on the server (e.g. switching between git branches).
+        operation
+            .check_if_nodes_synced(&updated_nodes, sync_progress_tx)
+            .await?;
+        Ok(operation)
+    }
+
+    pub async fn flush_nodes_pending_sync(
+        mut self,
+        repo_metadata: &RepoMetadata,
+        root_node_hash: NodeHash,
+        mapping_updates: &LeafToFragmentMetadataMapping,
+        sync_progress_tx: async_channel::Sender<SyncProgress>,
+    ) -> Result<FlushFragmentResult, SyncOperationError> {
+        let mut leaves = Vec::new();
+        let mut intermediate_nodes = Vec::new();
+
+        for node in mem::take(&mut self.nodes_pending_sync) {
+            if node.is_leaf() {
+                leaves.push(node);
+            } else {
+                intermediate_nodes.push(node);
+            }
+        }
+
+        let mut failed_to_sync_nodes: HashSet<NodeHash> = HashSet::new();
+        let mut files_need_resync = ChangedFiles::default();
+        let mut total_fragment_count = 0;
+        let mut total_fragment_size_bytes = 0;
+
+        let total_nodes_to_sync = leaves.len() + intermediate_nodes.len();
+        let mut completed_nodes = 0;
+
+        let leaf_batches = batch_leaves_by_size(
+            &leaves,
+            mapping_updates,
+            self.embedding_generation_batch_size,
+            MAX_BATCH_CONTENT_BYTES,
+        )
+        .map_err(SyncOperationError::Other)?;
+
+        for chunk in &leaf_batches {
+            let mut fragment_metadatas = HashMap::new();
+
+            for node in chunk {
+                let content_hash = node.content_hash().expect("Node should be leaf");
+                let metadatas = mapping_updates
+                    .get(content_hash.as_ref())
+                    .ok_or(anyhow!("Couldn't find metadata for hash"))?;
+
+                for metadata in metadatas {
+                    fragment_metadatas.insert(content_hash.clone(), metadata.clone());
+                }
+            }
+
+            let fragment_metadata_clone = fragment_metadatas.clone();
+            let res = build_fragments_from_metadata(fragment_metadata_clone.into_iter()).await;
+
+            if !res.fail_to_read.is_empty() {
+                let failed_node_count = res.fail_to_read.len();
+                failed_to_sync_nodes.extend(
+                    res.fail_to_read
+                        .into_iter()
+                        .map(|content_hash| content_hash.into()),
+                );
+                files_need_resync.add_paths(res.fail_to_read_path).await;
+                log::warn!("Failed to read {failed_node_count} fragments from disk");
+            }
+
+            // Add retry for embedding generation
+            let fragments_to_sync = res.successfully_read;
+
+            // Track fragment metrics
+            total_fragment_count += fragments_to_sync.len();
+            total_fragment_size_bytes += fragments_to_sync
+                .iter()
+                .map(|f| f.content.len())
+                .sum::<usize>();
+
+            let rx = self
+                .sync_queue
+                .enqueue_with_result(
+                    SyncTask::GenerateEmbeddings(GenerateEmbeddingsTask {
+                        store_client: self.store_client.clone(),
+                        embedding_config: self.embedding_config,
+                        fragments: fragments_to_sync,
+                        root_node_hash: root_node_hash.clone(),
+                        repo_metadata: repo_metadata.clone(),
+                    }),
+                    None,
+                    "generate_embeddings".to_string(),
+                )
+                .await;
+
+            let Ok(task_result) = rx.await else {
+                return Err(SyncOperationError::Other(anyhow::anyhow!(
+                    "Sync queue task cancelled"
+                )));
+            };
+
+            let res = match task_result.inspect(|res| {
+                if let SyncQueueResult::GenerateEmbeddings(res) = res {
+                    let failed_fragments = res
+                        .iter()
+                        .filter_map(|(hash, &success)| {
+                            if !success {
+                                failed_to_sync_nodes.insert(hash.into());
+                                fragment_metadatas.remove(hash)
+                            } else {
+                                None
+                            }
+                        })
+                        .collect_vec();
+                    if !failed_fragments.is_empty() {
+                        log::warn!(
+                            "Failed to generate embeddings for some hashes:\n{failed_fragments:#?}"
+                        );
+                    }
+                }
+            }) {
+                Ok(res) => res,
+                Err(err) => {
+                    log::error!("Failed to generate embeddings: {err:?}");
+                    if files_need_resync.is_empty() {
+                        return Err(SyncOperationError::ServerSyncError(err));
+                    } else {
+                        return Err(SyncOperationError::ReadFragmentError(files_need_resync));
+                    }
+                }
+            };
+
+            completed_nodes += chunk.len();
+            let _ = sync_progress_tx.try_send(SyncProgress::Syncing {
+                completed_nodes: completed_nodes.saturating_sub(failed_to_sync_nodes.len()),
+                total_nodes: total_nodes_to_sync,
+            });
+
+            log::debug!("Generated embedding for the following nodes: {res:?}");
+        }
+
+        for chunk in intermediate_nodes.chunks(self.embedding_generation_batch_size) {
+            let nodes_to_sync: Vec<IntermediateNode> = chunk
+                .iter()
+                .filter_map(|node| {
+                    let outdated = node
+                        .children()
+                        .any(|child| failed_to_sync_nodes.contains(&child.hash()));
+                    if outdated {
+                        failed_to_sync_nodes.insert(node.hash());
+                        None
+                    } else {
+                        Some(IntermediateNode {
+                            hash: node.hash(),
+                            children: node.children().map(|child| child.hash()).collect(),
+                        })
+                    }
+                })
+                .collect();
+
+            let rx = self
+                .sync_queue
+                .enqueue_with_result(
+                    SyncTask::UpdateIntermediateNodes(UpdateIntermediateNodesTask {
+                        store_client: self.store_client.clone(),
+                        embedding_config: self.embedding_config,
+                        nodes: nodes_to_sync.clone(),
+                    }),
+                    None,
+                    "update intermediate nodes".to_string(),
+                )
+                .await;
+
+            let Ok(task_result) = rx.await else {
+                return Err(SyncOperationError::Other(anyhow::anyhow!(
+                    "Sync queue task cancelled"
+                )));
+            };
+
+            let update_result = match task_result.inspect(|res| {
+                if let SyncQueueResult::UpdateIntermediateNodes(res) = res {
+                    let failed_nodes = res
+                        .iter()
+                        .filter_map(|(hash, success)| {
+                            if !success {
+                                failed_to_sync_nodes.insert(hash.clone());
+                                Some(hash.clone())
+                            } else {
+                                None
+                            }
+                        })
+                        .collect_vec();
+
+                    if !failed_nodes.is_empty() {
+                        log::warn!("Failed to sync some intermediate nodes");
+                    }
+                }
+            }) {
+                Ok(res) => res,
+                Err(err) => {
+                    log::error!("Failed to sync intermediate node: {err:?}");
+                    if files_need_resync.is_empty() {
+                        return Err(SyncOperationError::ServerSyncError(err));
+                    } else {
+                        return Err(SyncOperationError::ReadFragmentError(files_need_resync));
+                    }
+                }
+            };
+
+            completed_nodes += chunk.len();
+            let _ = sync_progress_tx.try_send(SyncProgress::Syncing {
+                completed_nodes: completed_nodes.saturating_sub(failed_to_sync_nodes.len()),
+                total_nodes: total_nodes_to_sync,
+            });
+
+            log::debug!("Updated the following nodes: {update_result:?}");
+        }
+
+        if !failed_to_sync_nodes.is_empty() {
+            log::warn!(
+                "Failed to sync {} nodes to the server for root {}",
+                failed_to_sync_nodes.len(),
+                root_node_hash
+            );
+
+            if files_need_resync.is_empty() {
+                return Err(SyncOperationError::ServerSyncError(Error::Other(
+                    anyhow::anyhow!("Failed to sync some nodes to the server"),
+                )));
+            } else {
+                return Err(SyncOperationError::ReadFragmentError(files_need_resync));
+            }
+        }
+
+        log::info!("Successfully flushed nodes pending sync for root {root_node_hash}");
+        Ok(FlushFragmentResult {
+            fragment_count: total_fragment_count,
+            total_fragment_size_bytes,
+        })
+    }
+
+    pub fn total_pending_nodes_count(&self) -> usize {
+        self.nodes_pending_sync.len()
+    }
+
+    async fn check_if_nodes_synced(
+        &mut self,
+        nodes_to_check: &[NodeLens<'a>],
+        sync_progress_tx: async_channel::Sender<SyncProgress>,
+    ) -> Result<Vec<NodeLens<'a>>, SyncOperationError> {
+        let chunks = nodes_to_check.chunks(SYNC_NODE_BATCH_SIZE);
+        let mut res = Vec::new();
+
+        for chunk in chunks {
+            let mut node_hashes = Vec::new();
+
+            for node in chunk {
+                node_hashes.push(node.hash());
+            }
+
+            let rx = self
+                .sync_queue
+                .enqueue_with_result(
+                    SyncTask::SyncMerkleTree(SyncMerkleTreeTask {
+                        store_client: self.store_client.clone(),
+                        embedding_config: self.embedding_config,
+                        nodes: node_hashes.clone(),
+                    }),
+                    None,
+                    "update intermediate nodes".to_string(),
+                )
+                .await;
+
+            let nodes_need_sync = match rx.await {
+                Ok(Ok(SyncQueueResult::SyncMerkleTree(res))) => res,
+                Ok(Ok(_)) => {
+                    return Err(SyncOperationError::Other(anyhow::anyhow!(
+                        "Shouldn't receive other task result in channel"
+                    )))
+                }
+                Ok(Err(e)) => return Err(SyncOperationError::ServerSyncError(e)),
+                Err(_) => {
+                    return Err(SyncOperationError::Other(anyhow::anyhow!(
+                        "Sync queue task cancelled"
+                    )))
+                }
+            };
+
+            // Iterate over the nodes that need to be synced and add their children to the next queue.
+            for node in chunk {
+                if nodes_need_sync.contains(&node.hash()) {
+                    self.nodes_pending_sync.push(*node);
+                    res.extend(node.children());
+                }
+            }
+        }
+
+        let _ = sync_progress_tx.try_send(SyncProgress::Discovering {
+            total_nodes: self.nodes_pending_sync.len(),
+        });
+
+        Ok(res)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(super) enum SyncOperationError {
+    #[error("Error reading some fragments {0:#?}")]
+    ReadFragmentError(ChangedFiles),
+    #[error("Error syncing nodes with server {0:#}")]
+    ServerSyncError(Error),
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+
+/// Partitions `leaves` into batches where each batch contains at most `max_count` leaves
+/// AND at most `max_bytes` of estimated content.
+///
+/// Content size for each leaf is estimated from the `byte_range` in its `FragmentMetadata`,
+/// which is available without reading from disk.
+fn batch_leaves_by_size<'a>(
+    leaves: &[NodeLens<'a>],
+    mapping_updates: &LeafToFragmentMetadataMapping,
+    max_count: usize,
+    max_bytes: usize,
+) -> Result<Vec<Vec<NodeLens<'a>>>> {
+    if leaves.is_empty() {
+        return Ok(vec![]);
+    }
+
+    let mut batches = Vec::new();
+    let mut current_batch = Vec::new();
+    let mut current_bytes: usize = 0;
+
+    for leaf in leaves {
+        let content_hash = leaf.content_hash().expect("Node should be leaf");
+        let metadatas = mapping_updates
+            .get(content_hash.as_ref())
+            .ok_or_else(|| anyhow!("Couldn't find metadata for hash {content_hash:?}"))?;
+
+        let leaf_bytes: usize = metadatas.iter().map(|m| m.content_byte_size()).sum();
+
+        // If the current batch is non-empty and adding this leaf would exceed either limit,
+        // finalize the current batch and start a new one.
+        if !current_batch.is_empty()
+            && (current_batch.len() >= max_count || current_bytes + leaf_bytes > max_bytes)
+        {
+            batches.push(std::mem::take(&mut current_batch));
+            current_bytes = 0;
+        }
+
+        current_batch.push(*leaf);
+        current_bytes += leaf_bytes;
+    }
+
+    if !current_batch.is_empty() {
+        batches.push(current_batch);
+    }
+
+    Ok(batches)
+}
+
+impl IsTransientError for Error {
+    fn is_transient(&self) -> bool {
+        // TODO: match on the error type and only return true of actual transient error.
+        true
+    }
+}
+
+impl From<SyncOperationError> for Error {
+    fn from(error: SyncOperationError) -> Self {
+        match error {
+            SyncOperationError::Other(e) => Self::Other(e),
+            SyncOperationError::ServerSyncError(e) => e,
+            SyncOperationError::ReadFragmentError(_) => Self::FileSystemStateChanged,
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "sync_client_tests.rs"]
+mod tests;

crates/ai/src/index/full_source_code_embedding/sync_client_tests.rs

@@ -0,0 +1,112 @@
+use std::collections::HashMap;
+
+use futures::executor::block_on;
+
+use super::batch_leaves_by_size;
+use crate::index::full_source_code_embedding::merkle_tree::{construct_test_merkle_tree, NodeLens};
+
+use virtual_fs::VirtualFS;
+
+/// Collect all leaf nodes from a merkle tree by walking it recursively.
+fn collect_leaves<'a>(node: NodeLens<'a>) -> Vec<NodeLens<'a>> {
+    if node.is_leaf() {
+        return vec![node];
+    }
+    node.children().flat_map(collect_leaves).collect()
+}
+
+#[test]
+fn test_batch_leaves_single_batch_when_under_limits() {
+    VirtualFS::test("batch_single", |dirs, mut sandbox| {
+        let (tree, metadata) = block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+
+        let leaves = collect_leaves(tree.root_node());
+        assert!(!leaves.is_empty(), "Tree should have leaf nodes");
+
+        let batches = batch_leaves_by_size(&leaves, metadata.mapping(), 1000, 10_000_000).unwrap();
+
+        assert_eq!(batches.len(), 1, "All leaves should fit in a single batch");
+        assert_eq!(
+            batches[0].len(),
+            leaves.len(),
+            "The single batch should contain all leaves"
+        );
+    });
+}
+
+#[test]
+fn test_batch_leaves_splits_on_count_limit() {
+    VirtualFS::test("batch_count", |dirs, mut sandbox| {
+        let (tree, metadata) = block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+
+        let leaves = collect_leaves(tree.root_node());
+        let leaf_count = leaves.len();
+        assert!(leaf_count >= 2, "Need at least 2 leaves for this test");
+
+        let batches = batch_leaves_by_size(&leaves, metadata.mapping(), 1, 10_000_000).unwrap();
+
+        assert_eq!(
+            batches.len(),
+            leaf_count,
+            "Each leaf should be in its own batch when max_count=1"
+        );
+        for batch in &batches {
+            assert_eq!(batch.len(), 1);
+        }
+    });
+}
+
+#[test]
+fn test_batch_leaves_splits_on_byte_limit() {
+    VirtualFS::test("batch_bytes", |dirs, mut sandbox| {
+        let (tree, metadata) = block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+
+        let leaves = collect_leaves(tree.root_node());
+        let leaf_count = leaves.len();
+        assert!(leaf_count >= 2, "Need at least 2 leaves for this test");
+
+        // max_bytes=1 means every leaf exceeds the limit, but progress guarantee
+        // ensures each still gets its own batch.
+        let batches = batch_leaves_by_size(&leaves, metadata.mapping(), 1000, 1).unwrap();
+
+        assert_eq!(
+            batches.len(),
+            leaf_count,
+            "Each leaf should be in its own batch when max_bytes=1 (progress guarantee)"
+        );
+        for batch in &batches {
+            assert_eq!(batch.len(), 1);
+        }
+    });
+}
+
+#[test]
+fn test_batch_leaves_empty_input() {
+    let leaves: Vec<NodeLens<'_>> = vec![];
+    let mapping = HashMap::new();
+
+    let batches = batch_leaves_by_size(&leaves, &mapping, 100, 4_000_000).unwrap();
+    assert!(
+        batches.is_empty(),
+        "Empty input should produce empty output"
+    );
+}
+
+#[test]
+fn test_batch_leaves_missing_metadata_returns_error() {
+    VirtualFS::test("batch_missing", |dirs, mut sandbox| {
+        let (tree, _metadata) = block_on(construct_test_merkle_tree(&dirs, &mut sandbox));
+
+        let leaves = collect_leaves(tree.root_node());
+        assert!(!leaves.is_empty());
+
+        // Pass an empty mapping — every leaf lookup should fail.
+        let empty_mapping = HashMap::new();
+        let result = batch_leaves_by_size(&leaves, &empty_mapping, 1000, 10_000_000);
+
+        assert!(
+            result.is_err(),
+            "Should return error when metadata is missing"
+        );
+    });
+}

crates/ai/src/index/locations.rs

@@ -0,0 +1,34 @@
+use serde::{Deserialize, Serialize};
+use std::{ops::Range, path::PathBuf};
+
+/// A line-based file fragment location.
+///
+/// Represents a specific portion of a file by its path and line range.
+/// Used for passing precise code context fragments between components.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct FileFragmentLocation {
+    /// The absolute path to the file
+    pub path: PathBuf,
+    /// The line range (inclusive start, inclusive end)
+    pub line_ranges: Vec<Range<usize>>,
+}
+
+/// Combined representation of a file context, which can be either
+/// a whole file or a specific fragment of a file.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum CodeContextLocation {
+    /// Represent an entire file (used in outline-based context)
+    WholeFile(PathBuf),
+    /// Represent a specific fragment of a file (used with FullSourceCodeEmbedding)
+    Fragment(FileFragmentLocation),
+}
+
+impl CodeContextLocation {
+    /// Get the file path regardless of whether this is a whole file or fragment
+    pub fn path(&self) -> &PathBuf {
+        match self {
+            CodeContextLocation::WholeFile(path) => path,
+            CodeContextLocation::Fragment(fragment) => &fragment.path,
+        }
+    }
+}

crates/ai/src/index/mod.rs

@@ -0,0 +1,55 @@
+mod file_outline;
+pub mod locations;
+pub const DEFAULT_SYNC_REQUESTS_PER_MIN: u32 = 600;
+
+#[allow(dead_code)]
+pub mod full_source_code_embedding;
+
+#[cfg(feature = "local_fs")]
+pub use file_outline::build_outline;
+
+pub use file_outline::{Outline, Symbol};
+pub use repo_metadata::{BuildTreeError, DirectoryEntry, Entry, FileId, FileMetadata};
+
+cfg_if::cfg_if! {
+    if #[cfg(feature = "local_fs")] {
+        pub use repo_metadata::{
+            matches_gitignores, path_passes_filters,
+        };
+    }
+}
+
+#[cfg(feature = "local_fs")]
+use native::*;
+#[cfg(not(feature = "local_fs"))]
+use wasm::*;
+
+#[cfg(feature = "local_fs")]
+mod native {
+    use std::thread::available_parallelism;
+
+    pub(super) const MAX_PARALLEL_THREADS: usize = 2;
+
+    fn create_thread_pool() -> Option<rayon::ThreadPool> {
+        let num_threads = available_parallelism()
+            .map(|parallelism| (parallelism.get() / 2).clamp(1, MAX_PARALLEL_THREADS))
+            .unwrap_or(MAX_PARALLEL_THREADS);
+
+        rayon::ThreadPoolBuilder::new()
+            .thread_name(|index| format!("warp-code-indexing-{index}"))
+            .num_threads(num_threads)
+            .build()
+            .ok()
+    }
+
+    lazy_static::lazy_static! {
+        pub(super) static ref THREADPOOL: Option<rayon::ThreadPool> = create_thread_pool();
+    }
+}
+
+#[cfg(not(feature = "local_fs"))]
+mod wasm {
+    lazy_static::lazy_static! {
+        pub(super) static ref THREADPOOL: Option<rayon::ThreadPool> = None;
+    }
+}

crates/ai/src/lib.rs

@@ -0,0 +1,15 @@
+pub mod agent;
+pub mod api_keys;
+pub mod aws_credentials;
+pub mod llm_id;
+
+pub use llm_id::LLMId;
+pub mod diff_validation;
+pub mod document;
+pub mod gfm_table;
+pub mod index;
+pub mod paths;
+pub mod project_context;
+pub mod skills;
+mod telemetry;
+pub mod workspace;

crates/ai/src/llm_id.rs

@@ -0,0 +1,35 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct LLMId(String);
+
+impl LLMId {
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl From<String> for LLMId {
+    fn from(value: String) -> Self {
+        Self(value)
+    }
+}
+
+impl From<&str> for LLMId {
+    fn from(value: &str) -> Self {
+        value.to_owned().into()
+    }
+}
+
+impl From<LLMId> for String {
+    fn from(value: LLMId) -> Self {
+        value.0
+    }
+}
+
+impl std::fmt::Display for LLMId {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}

crates/ai/src/paths.rs

@@ -0,0 +1,125 @@
+use typed_path::{TypedPath, TypedPathBuf, WindowsPath};
+use warp_terminal::shell::ShellLaunchData;
+use warp_util::path::{
+    convert_msys2_to_windows_native_path, convert_wsl_to_windows_host_path, msys2_exe_to_root,
+};
+use warpui::platform::OperatingSystem;
+
+fn use_unix_paths(shell: Option<&ShellLaunchData>) -> bool {
+    OperatingSystem::get().is_linux()
+        || OperatingSystem::get().is_mac()
+        || shell.is_some_and(|shell| {
+            matches!(
+                shell,
+                ShellLaunchData::WSL { .. } | ShellLaunchData::MSYS2 { .. }
+            )
+        })
+}
+
+pub fn join_paths(paths: &[&str], shell: Option<&ShellLaunchData>) -> String {
+    let use_unix_paths = use_unix_paths(shell);
+
+    let base_path = if use_unix_paths {
+        TypedPathBuf::unix()
+    } else {
+        TypedPathBuf::windows()
+    };
+    paths
+        .iter()
+        .fold(base_path, |acc, path| acc.join(path))
+        .to_string_lossy()
+        .into_owned()
+}
+
+fn shell_native_absolute_path_internal(
+    file_path: &str,
+    shell: Option<&ShellLaunchData>,
+    current_working_directory: &str,
+) -> TypedPathBuf {
+    let expanded_path = shellexpand::tilde(file_path).into_owned();
+
+    let use_unix_paths = use_unix_paths(shell);
+    let (cwd, file_path) = if use_unix_paths {
+        (
+            TypedPathBuf::from_unix(current_working_directory),
+            TypedPath::unix(&expanded_path),
+        )
+    } else {
+        (
+            TypedPathBuf::from_windows(current_working_directory),
+            TypedPath::windows(&expanded_path),
+        )
+    };
+    cwd.join(file_path).normalize()
+}
+
+/// Returns the absolute path of the path in the shell's native format.
+///
+/// On Unix systems, this will always be Unix encoded paths. On Windows, this
+/// will be a Windows encoded path unless the user is using WSL or Git Bash, in
+/// which case Unix encoded paths will be used.
+pub fn shell_native_absolute_path(
+    file_path: &str,
+    shell: Option<&ShellLaunchData>,
+    current_working_directory: Option<&String>,
+) -> String {
+    let Some(cwd) = current_working_directory else {
+        return shellexpand::tilde(file_path).into_owned();
+    };
+    shell_native_absolute_path_internal(file_path, shell, cwd)
+        .to_string_lossy()
+        .into_owned()
+}
+
+/// Returns the absolute path of the path in the host's native format.
+///
+/// This should be used over [`shell_native_absolute_path`] when we need an
+/// absolute path in the format of the user's OS, regardless of what shell
+/// they're using. e.g. A Windows encoded path when the user is using WSL.
+pub fn host_native_absolute_path(
+    file_path: &str,
+    shell: &Option<ShellLaunchData>,
+    current_working_directory: &Option<String>,
+) -> String {
+    let Some(cwd) = current_working_directory.as_ref() else {
+        return shellexpand::tilde(file_path).into_owned();
+    };
+    let normalized_path = shell_native_absolute_path_internal(file_path, shell.as_ref(), cwd);
+
+    match shell {
+        Some(ShellLaunchData::WSL { distro }) => {
+            match convert_wsl_to_windows_host_path(&normalized_path.to_path(), distro) {
+                Ok(path) => path.to_string_lossy().into_owned(),
+                Err(err) => {
+                    log::error!(
+                        "Could not convert WSL to Windows host path {normalized_path:?}: {err:#}"
+                    );
+                    normalized_path.to_string_lossy().into_owned()
+                }
+            }
+        }
+        Some(ShellLaunchData::MSYS2 {
+            executable_path, ..
+        }) => {
+            match convert_msys2_to_windows_native_path(
+                &normalized_path.to_path(),
+                &msys2_exe_to_root(WindowsPath::new(
+                    executable_path.as_os_str().as_encoded_bytes(),
+                )),
+            ) {
+                Ok(path) => path.to_string_lossy().into_owned(),
+                Err(err) => {
+                    log::error!(
+                        "Could not convert MSYS2 to Windows host path {normalized_path:?}: {err:#}"
+                    );
+                    normalized_path.to_string_lossy().into_owned()
+                }
+            }
+        }
+        _ => normalized_path.to_string_lossy().into_owned(),
+    }
+}
+
+#[cfg(test)]
+#[path = "paths_tests.rs"]
+mod tests;

crates/ai/src/paths_tests.rs

@@ -0,0 +1,370 @@
+#[cfg(windows)]
+use std::path::PathBuf;
+
+#[cfg(windows)]
+use warp_terminal::shell::{ShellLaunchData, ShellType};
+
+use super::*;
+
+#[cfg(unix)]
+#[test]
+fn test_host_native_absolute_path() {
+    // Test with absolute path
+    assert_eq!(
+        host_native_absolute_path(
+            "/home/user/file.txt",
+            &None,
+            &Some("/current/dir".to_string())
+        ),
+        "/home/user/file.txt"
+    );
+
+    // Test with relative path
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &Some("/current/dir".to_string())),
+        "/current/dir/file.txt"
+    );
+
+    // Test with tilde expansion
+    assert_eq!(
+        host_native_absolute_path("~/file.txt", &None, &Some("/current/dir".to_string())),
+        shellexpand::tilde("~/file.txt").into_owned()
+    );
+
+    // Test with ..
+    assert_eq!(
+        host_native_absolute_path("../user/file.txt", &None, &Some("/current/dir".to_string())),
+        "/current/user/file.txt"
+    );
+
+    // Test with .
+    assert_eq!(
+        host_native_absolute_path("./user/file.txt", &None, &Some("/current/dir".to_string())),
+        "/current/dir/user/file.txt"
+    );
+
+    // Test with no current working directory
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &None),
+        "file.txt"
+    );
+
+    // Test with empty current working directory
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &Some("".to_string())),
+        "file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_host_native_absolute_path() {
+    // Test with absolute path
+    assert_eq!(
+        host_native_absolute_path(
+            r"C:\home\user\file.txt",
+            &None,
+            &Some(r"C:\current\dir".to_string())
+        ),
+        r"C:\home\user\file.txt"
+    );
+
+    // Test with relative path
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &Some(r"C:\current\dir".to_string())),
+        r"C:\current\dir\file.txt"
+    );
+
+    // Test with tilde expansion
+    assert_eq!(
+        host_native_absolute_path(r"~\file.txt", &None, &Some(r"C:\current\dir".to_string())),
+        shellexpand::tilde(r"~\file.txt").into_owned()
+    );
+
+    // Test with ..
+    assert_eq!(
+        host_native_absolute_path(
+            r"..\user\file.txt",
+            &None,
+            &Some(r"C:\current\dir".to_string())
+        ),
+        r"C:\current\user\file.txt"
+    );
+
+    // Test with .
+    assert_eq!(
+        host_native_absolute_path(
+            r".\user\file.txt",
+            &None,
+            &Some(r"C:\current\dir".to_string())
+        ),
+        r"C:\current\dir\user\file.txt"
+    );
+
+    // Test with no current working directory
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &None),
+        "file.txt"
+    );
+
+    // Test with empty current working directory
+    assert_eq!(
+        host_native_absolute_path("file.txt", &None, &Some("".to_string())),
+        "file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_git_bash_paths() {
+    let executable_path = PathBuf::from(r"C:\Program Files\Git\usr\bin\bash.exe");
+    let git_bash_shell = Some(ShellLaunchData::MSYS2 {
+        executable_path,
+        shell_type: ShellType::Bash,
+    });
+
+    assert_eq!(
+        host_native_absolute_path(
+            "/c/Users/username/project/file.txt",
+            &git_bash_shell,
+            &Some("/c/Users/username".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+
+    assert_eq!(
+        host_native_absolute_path(
+            "project/file.txt",
+            &git_bash_shell,
+            &Some("/c/Users/username".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+
+    assert_eq!(
+        host_native_absolute_path(
+            "../project/file.txt",
+            &git_bash_shell,
+            &Some("/c/Users/username/docs".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_wsl_paths() {
+    let wsl_shell = Some(ShellLaunchData::WSL {
+        distro: "Ubuntu".to_string(),
+    });
+
+    assert_eq!(
+        host_native_absolute_path(
+            "/mnt/c/Users/username/project/file.txt",
+            &wsl_shell,
+            &Some("/mnt/c/Users/username".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+
+    assert_eq!(
+        host_native_absolute_path(
+            "project/file.txt",
+            &wsl_shell,
+            &Some("/mnt/c/Users/username".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+
+    assert_eq!(
+        host_native_absolute_path(
+            "../project/file.txt",
+            &wsl_shell,
+            &Some("/mnt/c/Users/username/docs".to_string())
+        ),
+        r"c:\Users\username\project\file.txt"
+    );
+
+    assert_eq!(
+        host_native_absolute_path(
+            "/home/user/file.txt",
+            &wsl_shell,
+            &Some("/mnt/c/Users/username".to_string())
+        ),
+        r"\\WSL$\Ubuntu\home\user\file.txt"
+    );
+}
+
+#[cfg(unix)]
+#[test]
+fn test_shell_native_absolute_path() {
+    // Test with absolute path
+    let cwd = Some("/current/dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("/home/user/file.txt", None, cwd.as_ref()),
+        "/home/user/file.txt"
+    );
+
+    // Test with relative path
+    let cwd = Some("/current/dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, cwd.as_ref()),
+        "/current/dir/file.txt"
+    );
+
+    // Test with tilde expansion
+    let cwd = Some("/current/dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("~/file.txt", None, cwd.as_ref()),
+        shellexpand::tilde("~/file.txt").into_owned()
+    );
+
+    // Test with ..
+    let cwd = Some("/current/dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("../user/file.txt", None, cwd.as_ref()),
+        "/current/user/file.txt"
+    );
+
+    // Test with .
+    let cwd = Some("/current/dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("./user/file.txt", None, cwd.as_ref()),
+        "/current/dir/user/file.txt"
+    );
+
+    // Test with no current working directory
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, None),
+        "file.txt"
+    );
+
+    // Test with empty current working directory
+    let cwd = Some("".to_string());
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, cwd.as_ref()),
+        "file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_shell_native_absolute_path() {
+    // Test with absolute path
+    let cwd = Some(r"C:\current\dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path(r"C:\home\user\file.txt", None, cwd.as_ref()),
+        r"C:\home\user\file.txt"
+    );
+
+    // Test with relative path
+    let cwd = Some(r"C:\current\dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, cwd.as_ref()),
+        r"C:\current\dir\file.txt"
+    );
+
+    // Test with tilde expansion
+    let cwd = Some(r"C:\current\dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path(r"~\file.txt", None, cwd.as_ref()),
+        shellexpand::tilde(r"~\file.txt").into_owned()
+    );
+
+    // Test with ..
+    let cwd = Some(r"C:\current\dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path(r"..\user\file.txt", None, cwd.as_ref()),
+        r"C:\current\user\file.txt"
+    );
+
+    // Test with .
+    let cwd = Some(r"C:\current\dir".to_string());
+    assert_eq!(
+        shell_native_absolute_path(r".\user\file.txt", None, cwd.as_ref()),
+        r"C:\current\dir\user\file.txt"
+    );
+
+    // Test with no current working directory
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, None),
+        "file.txt"
+    );
+
+    // Test with empty current working directory
+    let cwd = Some("".to_string());
+    assert_eq!(
+        shell_native_absolute_path("file.txt", None, cwd.as_ref()),
+        "file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_shell_native_git_bash_paths() {
+    let executable_path = PathBuf::from(r"C:\Program Files\Git\usr\bin\bash.exe");
+    let git_bash_shell = Some(ShellLaunchData::MSYS2 {
+        executable_path,
+        shell_type: ShellType::Bash,
+    });
+
+    // In shell_native_absolute_path, MSYS2 paths should remain in Unix format
+    let cwd = Some("/c/Users/username".to_string());
+    assert_eq!(
+        shell_native_absolute_path(
+            "/c/Users/username/project/file.txt",
+            git_bash_shell.as_ref(),
+            cwd.as_ref()
+        ),
+        "/c/Users/username/project/file.txt"
+    );
+
+    let cwd = Some("/c/Users/username".to_string());
+    assert_eq!(
+        shell_native_absolute_path("project/file.txt", git_bash_shell.as_ref(), cwd.as_ref()),
+        "/c/Users/username/project/file.txt"
+    );
+
+    let cwd = Some("/c/Users/username/docs".to_string());
+    assert_eq!(
+        shell_native_absolute_path("../project/file.txt", git_bash_shell.as_ref(), cwd.as_ref()),
+        "/c/Users/username/project/file.txt"
+    );
+}
+
+#[cfg(windows)]
+#[test]
+fn test_shell_native_wsl_paths() {
+    let wsl_shell = Some(ShellLaunchData::WSL {
+        distro: "Ubuntu".to_string(),
+    });
+
+    // In shell_native_absolute_path, WSL paths should remain in Unix format
+    let cwd = Some("/mnt/c/Users/username".to_string());
+    assert_eq!(
+        shell_native_absolute_path(
+            "/mnt/c/Users/username/project/file.txt",
+            wsl_shell.as_ref(),
+            cwd.as_ref()
+        ),
+        "/mnt/c/Users/username/project/file.txt"
+    );
+
+    let cwd = Some("/mnt/c/Users/username".to_string());
+    assert_eq!(
+        shell_native_absolute_path("project/file.txt", wsl_shell.as_ref(), cwd.as_ref()),
+        "/mnt/c/Users/username/project/file.txt"
+    );
+
+    let cwd = Some("/mnt/c/Users/username/docs".to_string());
+    assert_eq!(
+        shell_native_absolute_path("../project/file.txt", wsl_shell.as_ref(), cwd.as_ref()),
+        "/mnt/c/Users/username/project/file.txt"
+    );
+
+    let cwd = Some("/mnt/c/Users/username".to_string());
+    assert_eq!(
+        shell_native_absolute_path("/home/user/file.txt", wsl_shell.as_ref(), cwd.as_ref()),
+        "/home/user/file.txt"
+    );
+}

crates/ai/src/project_context/mod.rs

@@ -0,0 +1 @@
+pub mod model;

crates/ai/src/project_context/model.rs

@@ -0,0 +1,641 @@
+use anyhow::Result;
+#[cfg(feature = "local_fs")]
+use repo_metadata::repositories::RepoDetectionSource;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use warpui::{Entity, ModelContext, SingletonEntity};
+
+cfg_if::cfg_if! {
+    if #[cfg(feature = "local_fs")] {
+        use repo_metadata::entry::{Entry, FileMetadata};
+        use repo_metadata::repository::RepositorySubscriber;
+        use repo_metadata::{Repository, DirectoryWatcher, RepositoryUpdate};
+        use ignore::gitignore::Gitignore;
+        use async_channel::Sender;
+
+        const RULES_FILE_PATTERN: [&str; 2] = ["WARP.md", "AGENTS.md"];
+        const MAX_SCAN_DEPTH: usize = 3;
+        const MAX_FILES_TO_SCAN: usize = 5000;
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct ProjectRule {
+    pub path: PathBuf,
+    pub content: String,
+}
+
+#[derive(Debug, Default)]
+struct RuleAtPath {
+    parent_path: PathBuf,
+    warp_md: Option<ProjectRule>,
+    agents_md: Option<ProjectRule>,
+}
+
+impl RuleAtPath {
+    fn respected_rule(&self) -> Option<&ProjectRule> {
+        self.warp_md.as_ref().or(self.agents_md.as_ref())
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct ProjectRulesResult {
+    pub root_path: PathBuf,
+    pub active_rules: Vec<ProjectRule>,
+    pub additional_rule_paths: Vec<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ProjectRulePath {
+    pub path: PathBuf,
+    pub project_root: PathBuf,
+}
+
+struct FindRulesResult {
+    /// Rules that are active and should be eagerly applied.
+    active_rules: Vec<ProjectRule>,
+    /// Rule paths that are currently not active but available to be applied if
+    /// a file under its directory is edited.
+    available_rule_paths: Vec<String>,
+}
+
+#[cfg(feature = "local_fs")]
+fn matches_rules_pattern(file_name_str: &str) -> bool {
+    for pattern in RULES_FILE_PATTERN {
+        if file_name_str.to_lowercase() == pattern.to_lowercase() {
+            return true;
+        }
+    }
+    false
+}
+
+#[derive(Debug, Default)]
+struct ProjectRules {
+    rules: Vec<RuleAtPath>,
+}
+
+impl ProjectRules {
+    /// Finds the set of rules that are active in the given path and the set that are available to be applied.
+    fn find_active_or_applicable_rules(&self, path: &Path) -> FindRulesResult {
+        let mut active_rules = Vec::new();
+        let mut available_rule_paths = Vec::new();
+
+        // Collect all applicable rules (rules in directories that are ancestors of the target path)
+        for rule in &self.rules {
+            if let Some(respected_rule) = rule.respected_rule() {
+                // Check if the rule's directory is an ancestor of or equal to the target path
+                if path.starts_with(&rule.parent_path) {
+                    active_rules.push(respected_rule.clone());
+                } else {
+                    available_rule_paths.push(respected_rule.path.to_string_lossy().to_string());
+                }
+            }
+        }
+
+        FindRulesResult {
+            active_rules,
+            available_rule_paths,
+        }
+    }
+
+    /// Remove a rule from the set of project rules. This returns the removed rule.
+    #[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
+    fn remove_rule(&mut self, path: &Path) -> Option<ProjectRule> {
+        let parent = path.parent()?;
+        let file_name = path.file_name().and_then(|name| name.to_str())?;
+
+        let rule = self
+            .rules
+            .iter_mut()
+            .find(|rule| rule.parent_path == parent)?;
+
+        if file_name.to_lowercase() == "warp.md" {
+            rule.warp_md.take()
+        } else if file_name.to_lowercase() == "agents.md" {
+            rule.agents_md.take()
+        } else {
+            None
+        }
+    }
+
+    /// Upsert a rule to the set of project rules. This will create a new RuleAtPath entry if none exists and update the existin one
+    /// otherwise.
+    #[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
+    fn upsert_rule(&mut self, path: &Path, content: String) {
+        let Some(parent) = path.parent() else {
+            return;
+        };
+        let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
+            return;
+        };
+
+        let existing_rule = self
+            .rules
+            .iter_mut()
+            .find(|rule| rule.parent_path == parent);
+
+        let rule_file = Some(ProjectRule {
+            path: path.to_path_buf(),
+            content,
+        });
+
+        match existing_rule {
+            Some(rule) => {
+                if file_name.to_lowercase() == "warp.md" {
+                    rule.warp_md = rule_file;
+                } else if file_name.to_lowercase() == "agents.md" {
+                    rule.agents_md = rule_file;
+                }
+            }
+            None => {
+                let mut rule = RuleAtPath {
+                    parent_path: parent.to_path_buf(),
+                    ..Default::default()
+                };
+                if file_name.to_lowercase() == "warp.md" {
+                    rule.warp_md = rule_file;
+                } else if file_name.to_lowercase() == "agents.md" {
+                    rule.agents_md = rule_file;
+                }
+                self.rules.push(rule);
+            }
+        };
+    }
+}
+
+/// Singleton model that keeps track of mapping between paths and rule files
+/// Currently supports WARP.md files, but designed to be extensible
+#[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
+#[derive(Debug, Default)]
+pub struct ProjectContextModel {
+    /// Mapping from directory path to list of rule files found in that directory
+    path_to_rules: HashMap<PathBuf, ProjectRules>,
+}
+
+#[derive(Default, Debug)]
+pub struct RulesDelta {
+    pub discovered_rules: Vec<ProjectRulePath>,
+    pub deleted_rules: Vec<PathBuf>,
+}
+
+/// Events emitted by the ProjectContextModel
+pub enum ProjectContextModelEvent {
+    /// Emitted when a path has been indexed
+    PathIndexed,
+    /// Emitted when the known set of rule files changed
+    KnownRulesChanged(RulesDelta),
+}
+
+impl ProjectContextModel {
+    #[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
+    pub fn new_from_persisted(
+        persisted_rules: Vec<ProjectRulePath>,
+        ctx: &mut ModelContext<Self>,
+    ) -> Self {
+        #[cfg(feature = "local_fs")]
+        ctx.spawn(
+            async move { Self::read_persisted_rules(persisted_rules).await },
+            |me, mut res, ctx| {
+                for root in res.keys() {
+                    me.try_initialize_and_register_watcher(root, ctx);
+                }
+
+                // If we have any rules detected before fully loading the persisted rules, we want to
+                // keep the detected rules since it's more up to date.
+                res.extend(me.path_to_rules.drain());
+                me.path_to_rules = res;
+                ctx.emit(ProjectContextModelEvent::PathIndexed);
+            },
+        );
+
+        Self::default()
+    }
+
+    /// Index a path and find all rule files from that path up to the root directory
+    /// Returns a list of all rule files found
+    #[cfg_attr(not(feature = "local_fs"), allow(unused_variables))]
+    pub fn index_and_store_rules(
+        &mut self,
+        root_path: PathBuf,
+        ctx: &mut ModelContext<Self>,
+    ) -> Result<()> {
+        if self.path_to_rules.contains_key(&root_path) {
+            return Ok(());
+        }
+        #[cfg(feature = "local_fs")]
+        {
+            let root_clone = root_path.clone();
+
+            ctx.spawn(
+                async move { Self::scan_directory_for_rules(&root_path).await },
+                move |me, res: Result<ProjectRules>, ctx| match res {
+                    Ok(rule_files) => {
+                        me.register_watcher_for_path(&root_clone, ctx);
+
+                        // Persist the discovered rules.
+                        let delta = RulesDelta {
+                            discovered_rules: rule_files
+                                .rules
+                                .iter()
+                                .filter_map(|rule| {
+                                    rule.warp_md.as_ref().map(|rule| ProjectRulePath {
+                                        project_root: root_clone.clone(),
+                                        path: rule.path.clone(),
+                                    })
+                                })
+                                .chain(rule_files.rules.iter().filter_map(|rule| {
+                                    rule.agents_md.as_ref().map(|rule| ProjectRulePath {
+                                        project_root: root_clone.clone(),
+                                        path: rule.path.clone(),
+                                    })
+                                }))
+                                .collect(),
+                            deleted_rules: Default::default(),
+                        };
+                        ctx.emit(ProjectContextModelEvent::KnownRulesChanged(delta));
+
+                        me.path_to_rules.insert(root_clone, rule_files);
+                        ctx.emit(ProjectContextModelEvent::PathIndexed);
+                    }
+                    Err(e) => log::warn!(
+                        "Couldn't index rules for path {}: {}",
+                        root_clone.display(),
+                        e
+                    ),
+                },
+            );
+        }
+
+        Ok(())
+    }
+
+    /// This should be used when we are bootstrapping project rules from persisted rule paths. In this case,
+    /// the actual repo watcher might not have been registered yet. We will attempt to register that repo watcher
+    /// if it doesn't yet exists.
+    #[cfg(feature = "local_fs")]
+    fn try_initialize_and_register_watcher(&self, path: &Path, ctx: &mut ModelContext<Self>) {
+        use repo_metadata::repositories::DetectedRepositories;
+
+        let directory_watcher = DirectoryWatcher::handle(ctx);
+        if directory_watcher
+            .as_ref(ctx)
+            .get_watched_directory_for_path(path)
+            .is_some()
+        {
+            self.register_watcher_for_path(path, ctx);
+            return;
+        }
+
+        let fut = DetectedRepositories::handle(ctx).update(ctx, |model, ctx| {
+            model.detect_possible_git_repo(
+                &path.to_string_lossy(),
+                RepoDetectionSource::ProjectRulesIndexing,
+                ctx,
+            )
+        });
+
+        ctx.spawn(fut, move |me, repo_path_opt, ctx| {
+            if let Some(path) = repo_path_opt {
+                me.register_watcher_for_path(&path, ctx);
+            }
+        });
+    }
+
+    #[cfg(feature = "local_fs")]
+    fn register_watcher_for_path(&self, path: &Path, ctx: &mut ModelContext<Self>) {
+        let Some(repository_model) =
+            DirectoryWatcher::as_ref(ctx).get_watched_directory_for_path(path)
+        else {
+            return;
+        };
+
+        let (repository_update_tx, repository_update_rx) = async_channel::unbounded();
+        let start = repository_model.update(ctx, |repo, ctx| {
+            repo.start_watching(
+                Box::new(ProjectContextRepositorySubscriber {
+                    repository_update_tx,
+                }),
+                ctx,
+            )
+        });
+
+        let subscriber_id = start.subscriber_id;
+        let repository_model_for_cleanup = repository_model.downgrade();
+        let path_clone = path.to_path_buf();
+        let path_for_log = path_clone.clone();
+        ctx.spawn(start.registration_future, move |_, res, ctx| {
+            if let Err(err) = res {
+                log::warn!(
+                    "Failed to start watching repository for rule updates at {}: {err}",
+                    path_for_log.display()
+                );
+
+                if let Some(repository_model) = repository_model_for_cleanup.upgrade(ctx) {
+                    repository_model.update(ctx, |repo, ctx| {
+                        repo.stop_watching(subscriber_id, ctx);
+                    });
+                }
+            }
+        });
+
+        ctx.spawn_stream_local(
+            repository_update_rx.clone(),
+            move |me, update, ctx| {
+                if update.is_empty() {
+                    return;
+                }
+
+                let existing_rules = me.path_to_rules.remove(&path_clone);
+                let repo_path = path_clone.clone();
+                if let Some(rules) = existing_rules {
+                    let repo_path_for_closure = repo_path.clone();
+                    ctx.spawn(
+                        async move {
+                            Self::process_repository_updates(update, rules, repo_path).await
+                        },
+                        move |me, (rules, rule_delta), ctx| {
+                            ctx.emit(ProjectContextModelEvent::KnownRulesChanged(rule_delta));
+
+                            me.path_to_rules.insert(repo_path_for_closure, rules);
+                            ctx.emit(ProjectContextModelEvent::PathIndexed);
+                        },
+                    );
+                }
+            },
+            |_, _| {},
+        );
+    }
+
+    pub fn find_applicable_rules(&self, path: &Path) -> Option<ProjectRulesResult> {
+        let mut current_path = path.to_owned();
+        let mut active_rules = Vec::new();
+        let mut available_rule_paths = Vec::new();
+
+        // Find the root path with indexed rules and collect active rules
+        let mut found_rules = false;
+        loop {
+            if let Some(rules) = self.path_to_rules.get(&current_path) {
+                let result = rules.find_active_or_applicable_rules(path);
+
+                active_rules = result.active_rules;
+                available_rule_paths = result.available_rule_paths;
+
+                found_rules = true;
+                break;
+            }
+
+            if !current_path.pop() {
+                break;
+            }
+        }
+
+        if !found_rules {
+            return None;
+        }
+
+        if active_rules.is_empty() && available_rule_paths.is_empty() {
+            return None;
+        }
+
+        Some(ProjectRulesResult {
+            root_path: current_path,
+            active_rules,
+            additional_rule_paths: available_rule_paths,
+        })
+    }
+
+    #[cfg(feature = "local_fs")]
+    async fn process_repository_updates(
+        repository_update: RepositoryUpdate,
+        mut existing_rules: ProjectRules,
+        project_root: PathBuf,
+    ) -> (ProjectRules, RulesDelta) {
+        let mut rules_delta = RulesDelta::default();
+        // Handle deleted files - remove rules for deleted rule files
+        for target_file in &repository_update.deleted {
+            // Skip gitignored files
+            if target_file.is_ignored {
+                continue;
+            }
+            if let Some(file_name_str) = target_file.path.file_name().and_then(|name| name.to_str())
+            {
+                if matches_rules_pattern(file_name_str) {
+                    // Remove the rule from existing rules
+                    existing_rules.remove_rule(&target_file.path);
+                    rules_delta.deleted_rules.push(target_file.path.clone());
+
+                    log::debug!("Removed rule file: {}", target_file.path.display());
+                }
+            }
+        }
+
+        // Handle moved files - update paths for moved rule files
+        for (to_target, from_target) in &repository_update.moved {
+            // Skip gitignored files
+            if to_target.is_ignored || from_target.is_ignored {
+                continue;
+            }
+            if let Some(file_name_str) = to_target.path.file_name().and_then(|name| name.to_str()) {
+                if matches_rules_pattern(file_name_str) {
+                    // Find and update the rule with the old path
+                    if let Some(rule) = existing_rules.remove_rule(&from_target.path) {
+                        // Emit deletion event for old path
+                        rules_delta.deleted_rules.push(from_target.path.clone());
+
+                        existing_rules.upsert_rule(&to_target.path, rule.content);
+
+                        // Emit upsert event for new path
+                        rules_delta.discovered_rules.push(ProjectRulePath {
+                            path: to_target.path.clone(),
+                            project_root: project_root.clone(),
+                        });
+
+                        log::debug!(
+                            "Updated rule file path: {} -> {}",
+                            from_target.path.display(),
+                            to_target.path.display()
+                        );
+                    }
+                }
+            }
+        }
+
+        // Handle added/updated files - upsert rules for rule files
+        for target_file in repository_update.added_or_modified() {
+            // Skip gitignored files
+            if target_file.is_ignored {
+                continue;
+            }
+            if let Some(file_name_str) = target_file.path.file_name().and_then(|name| name.to_str())
+            {
+                if matches_rules_pattern(file_name_str) {
+                    // Read the content of the rule file
+                    match async_fs::read_to_string(&target_file.path).await {
+                        Ok(content) => {
+                            existing_rules.upsert_rule(&target_file.path, content);
+                        }
+                        Err(e) => {
+                            log::warn!(
+                                "Failed to read updated rule file {}: {}",
+                                target_file.path.display(),
+                                e
+                            );
+                        }
+                    }
+                }
+            }
+        }
+
+        (existing_rules, rules_delta)
+    }
+
+    /// Scan a directory for rule files (currently WARP.md, extensible for future file types)
+    /// Uses repo_metadata::entry::build_tree for efficient directory traversal
+    #[cfg(feature = "local_fs")]
+    async fn scan_directory_for_rules(dir_path: &Path) -> Result<ProjectRules> {
+        use repo_metadata::entry::IgnoredPathStrategy;
+
+        let mut rule_files = ProjectRules::default();
+
+        if !async_fs::metadata(dir_path).await?.is_dir() {
+            return Ok(rule_files);
+        }
+
+        // Use build_tree to collect all files, then filter for rule files
+        let mut files = Vec::<FileMetadata>::new();
+        let mut gitignores = Vec::<Gitignore>::new();
+
+        // Collect patterns that should not be ignored
+        let override_ignore_patterns: Vec<String> =
+            RULES_FILE_PATTERN.iter().map(|s| s.to_string()).collect();
+        let mut file_limit = MAX_FILES_TO_SCAN;
+
+        // Build the file tree using repo_metadata's build_tree function
+        let ignore_behavior = IgnoredPathStrategy::IncludeOnly(override_ignore_patterns.clone());
+
+        let _ = Entry::build_tree(
+            dir_path,
+            &mut files,
+            &mut gitignores,
+            Some(&mut file_limit),
+            MAX_SCAN_DEPTH,
+            0,
+            &ignore_behavior,
+        )?;
+
+        // Filter files to only include those matching RULES_FILE_PATTERN
+        for file_metadata in files {
+            let path = &file_metadata.path;
+            let file_name = path.file_name();
+
+            if let Some(file_name_str) = file_name {
+                if matches_rules_pattern(file_name_str) {
+                    // Read the content of the rule file
+                    let local_path = file_metadata.path.to_local_path_lossy();
+                    let content = match async_fs::read_to_string(&local_path).await {
+                        Ok(content) => content,
+                        Err(e) => {
+                            log::warn!("Failed to read rule file {}: {e}", file_metadata.path,);
+                            break;
+                        }
+                    };
+
+                    rule_files.upsert_rule(&local_path, content);
+                }
+            }
+        }
+
+        Ok(rule_files)
+    }
+
+    #[cfg(feature = "local_fs")]
+    async fn read_persisted_rules(
+        rule_paths: Vec<ProjectRulePath>,
+    ) -> HashMap<PathBuf, ProjectRules> {
+        let mut rules: HashMap<PathBuf, ProjectRules> = HashMap::new();
+
+        for rule in rule_paths {
+            match async_fs::read_to_string(&rule.path).await {
+                Ok(content) => {
+                    let existing_rules = rules.entry(rule.project_root).or_default();
+                    existing_rules.upsert_rule(&rule.path, content);
+                }
+                Err(e) => {
+                    log::debug!(
+                        "Failed to read rule file from persistence {}: {}",
+                        rule.path.display(),
+                        e
+                    );
+                    // Continue processing other files even if one fails
+                }
+            }
+        }
+
+        rules
+    }
+
+    pub fn indexed_rules(&self) -> impl Iterator<Item = PathBuf> + '_ {
+        self.path_to_rules.values().flat_map(|rules| {
+            rules.rules.iter().filter_map(|rules| {
+                rules
+                    .respected_rule()
+                    .map(|project_rule| project_rule.path.clone())
+            })
+        })
+    }
+
+    /// Returns the rule file paths associated with a specific workspace root path.
+    pub fn rules_for_workspace(&self, workspace_path: &Path) -> Vec<PathBuf> {
+        self.path_to_rules
+            .get(workspace_path)
+            .into_iter()
+            .flat_map(|rules| {
+                rules.rules.iter().filter_map(|rule| {
+                    rule.respected_rule()
+                        .map(|project_rule| project_rule.path.clone())
+                })
+            })
+            .collect()
+    }
+}
+
+impl Entity for ProjectContextModel {
+    type Event = ProjectContextModelEvent;
+}
+
+impl SingletonEntity for ProjectContextModel {}
+
+#[cfg(feature = "local_fs")]
+struct ProjectContextRepositorySubscriber {
+    repository_update_tx: Sender<RepositoryUpdate>,
+}
+
+#[cfg(feature = "local_fs")]
+impl RepositorySubscriber for ProjectContextRepositorySubscriber {
+    fn on_scan(
+        &mut self,
+        _repository: &Repository,
+        _ctx: &mut ModelContext<Repository>,
+    ) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
+        // The model can safely ignore the initial scan because the model only subscribes
+        // after the repository is already scanned.
+        Box::pin(async {})
+    }
+
+    fn on_files_updated(
+        &mut self,
+        _repository: &Repository,
+        update: &repo_metadata::RepositoryUpdate,
+        _ctx: &mut ModelContext<Repository>,
+    ) -> std::pin::Pin<Box<dyn std::prelude::rust_2024::Future<Output = ()> + Send + 'static>> {
+        let tx = self.repository_update_tx.clone();
+        let update = update.clone();
+        Box::pin(async move {
+            let _ = tx.send(update).await;
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "model_tests.rs"]
+mod tests;

crates/ai/src/project_context/model_tests.rs

@@ -0,0 +1,185 @@
+use super::*;
+use std::path::PathBuf;
+
+#[test]
+fn test_find_applicable_rules_empty_rules() {
+    let rules = ProjectRules { rules: vec![] };
+    let path = PathBuf::from("/a/b/c/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert!(result.is_empty());
+}
+
+#[test]
+fn test_find_applicable_rules_no_matching_rules() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/x/y/WARP.md"), "content1".to_string());
+    rules.upsert_rule(Path::new("/z/AGENTS.md"), "content2".to_string());
+
+    let path = PathBuf::from("/a/b/c/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert!(result.is_empty());
+}
+
+#[test]
+fn test_find_applicable_rules_single_matching_rule() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/WARP.md"), "content1".to_string());
+    rules.upsert_rule(Path::new("/x/AGENTS.md"), "content2".to_string());
+
+    let path = PathBuf::from("/a/b/c/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 1);
+    assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
+}
+
+#[test]
+fn test_find_applicable_rules_includes_all_ancestor_rules() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/WARP.md"), "root_warp".to_string());
+    rules.upsert_rule(Path::new("/a/b/WARP.md"), "nested_warp".to_string());
+    rules.upsert_rule(Path::new("/a/b/c/WARP.md"), "deep_warp".to_string());
+
+    let path = PathBuf::from("/a/b/c/d/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 3);
+
+    // All should be WARP.md files (same priority), order is not specified by depth
+    // Just verify all expected rules are present
+    let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
+    assert!(paths.contains(&PathBuf::from("/a/WARP.md")));
+    assert!(paths.contains(&PathBuf::from("/a/b/WARP.md")));
+    assert!(paths.contains(&PathBuf::from("/a/b/c/WARP.md")));
+}
+
+#[test]
+fn test_find_applicable_rules_multiple_patterns() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "agents_content".to_string());
+    rules.upsert_rule(Path::new("/a/WARP.md"), "warp_content".to_string());
+
+    let path = PathBuf::from("/a/b/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 2);
+
+    assert_eq!(result[0].path, PathBuf::from("/a/b/AGENTS.md"));
+    assert_eq!(result[0].content, "agents_content");
+    assert_eq!(result[1].path, PathBuf::from("/a/WARP.md"));
+    assert_eq!(result[1].content, "warp_content");
+}
+
+#[test]
+fn test_find_applicable_rules_exact_path_match() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/b/WARP.md"), "exact_match".to_string());
+
+    let path = PathBuf::from("/a/b/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 1);
+    assert_eq!(result[0].path, PathBuf::from("/a/b/WARP.md"));
+    assert_eq!(result[0].content, "exact_match");
+}
+
+#[test]
+fn test_find_applicable_rules_ignores_deeper_paths() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/WARP.md"), "applicable".to_string());
+    rules.upsert_rule(Path::new("/a/b/c/d/e/WARP.md"), "too_deep".to_string()); // Path doesn't contain /a/b
+
+    let path = PathBuf::from("/a/b/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 1);
+    assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
+    assert_eq!(result[0].content, "applicable");
+}
+
+#[test]
+fn test_find_applicable_rules_handles_root_path() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/WARP.md"), "root_rule".to_string());
+
+    let path = PathBuf::from("/a/b/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 1);
+    assert_eq!(result[0].path, PathBuf::from("/WARP.md"));
+    assert_eq!(result[0].content, "root_rule");
+}
+
+#[test]
+fn test_find_applicable_rules_complex_scenario() {
+    // This test covers the example from the original request:
+    // For path /a/b/c/file.rs with rules:
+    // - /a/WARP.md
+    // - /a/AGENTS.md
+    // - /a/b/WARP.md
+    // - /a/b/AGENTS.md
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/WARP.md"), "a_warp".to_string());
+    rules.upsert_rule(Path::new("/a/AGENTS.md"), "a_agents".to_string());
+    rules.upsert_rule(Path::new("/a/b/WARP.md"), "ab_warp".to_string());
+    rules.upsert_rule(Path::new("/a/b/AGENTS.md"), "ab_agents".to_string());
+    rules.upsert_rule(Path::new("/x/WARP.md"), "irrelevant".to_string()); // Should be ignored
+
+    let path = PathBuf::from("/a/b/c/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 2);
+
+    // Expect only WARP.md files to be included as they have higher priority.
+    assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
+    assert_eq!(result[0].content, "a_warp");
+    assert_eq!(result[1].path, PathBuf::from("/a/b/WARP.md"));
+    assert_eq!(result[1].content, "ab_warp");
+}
+
+#[test]
+fn test_find_applicable_rules_handles_unknown_file_patterns() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("/a/WARP.md"), "known_pattern".to_string());
+    rules.upsert_rule(Path::new("/a/UNKNOWN.md"), "unknown_pattern".to_string());
+    let path = PathBuf::from("/a/file.rs");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 1);
+
+    assert_eq!(result[0].path, PathBuf::from("/a/WARP.md"));
+    assert_eq!(result[0].content, "known_pattern");
+}
+
+#[test]
+fn test_find_applicable_rules_with_relative_paths() {
+    let mut rules = ProjectRules::default();
+
+    rules.upsert_rule(Path::new("src/WARP.md"), "src_warp".to_string());
+    rules.upsert_rule(
+        Path::new("src/components/WARP.md"),
+        "components_warp".to_string(),
+    );
+
+    let path = PathBuf::from("src/components/Button.tsx");
+
+    let result = rules.find_active_or_applicable_rules(&path).active_rules;
+    assert_eq!(result.len(), 2);
+
+    // Both are WARP.md files (same priority), order within same priority is not guaranteed
+    // Just verify both rules are present
+    let paths: Vec<PathBuf> = result.iter().map(|r| r.path.clone()).collect();
+    assert!(paths.contains(&PathBuf::from("src/WARP.md")));
+    assert!(paths.contains(&PathBuf::from("src/components/WARP.md")));
+}

crates/ai/src/skills/conversion.rs

@@ -0,0 +1,166 @@
+use std::path::PathBuf;
+
+use crate::{
+    agent::action_result::{AnyFileContent, FileContext},
+    skills::{ParsedSkill, SkillProvider, SkillScope},
+};
+use warp_multi_agent_api as api;
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum SkillConversionError {
+    #[error("No descriptor provided")]
+    MissingDescriptor,
+    #[error("No skill_reference provided")]
+    MissingReference,
+    #[error("No content provided")]
+    MissingContent,
+    #[error("Invalid scope")]
+    ScopeInvalid,
+    #[error("Invalid provider")]
+    ProviderInvalid,
+    #[error("Invalid content")]
+    ContentInvalid,
+}
+
+impl From<ParsedSkill> for api::Skill {
+    fn from(skill: ParsedSkill) -> Self {
+        api::Skill {
+            descriptor: Some(api::SkillDescriptor {
+                skill_reference: Some(api::skill_descriptor::SkillReference::Path(
+                    skill.path.to_string_lossy().to_string(),
+                )),
+                name: skill.name,
+                description: skill.description,
+                scope: Some(skill.scope.into()),
+                provider: Some(skill.provider.into()),
+            }),
+            content: Some(api::FileContent {
+                file_path: skill.path.to_string_lossy().to_string(),
+                content: skill.content,
+                line_range: skill
+                    .line_range
+                    .map(|line_range| api::FileContentLineRange {
+                        start: line_range.start as u32,
+                        end: line_range.end as u32,
+                    }),
+            }),
+        }
+    }
+}
+
+impl From<SkillScope> for api::skill_descriptor::Scope {
+    fn from(scope: SkillScope) -> Self {
+        let scope_type: api::skill_descriptor::scope::Type = match scope {
+            SkillScope::Home => api::skill_descriptor::scope::Type::Home(()),
+            SkillScope::Project => api::skill_descriptor::scope::Type::Project(()),
+            SkillScope::Bundled => api::skill_descriptor::scope::Type::Bundled(()),
+        };
+
+        api::skill_descriptor::Scope {
+            r#type: Some(scope_type),
+        }
+    }
+}
+
+impl From<SkillProvider> for api::skill_descriptor::Provider {
+    fn from(scope: SkillProvider) -> Self {
+        let provider_type: api::skill_descriptor::provider::Type = match scope {
+            SkillProvider::Warp => api::skill_descriptor::provider::Type::Warp(()),
+            SkillProvider::Agents => api::skill_descriptor::provider::Type::Agents(()),
+            SkillProvider::Claude => api::skill_descriptor::provider::Type::Claude(()),
+            SkillProvider::Codex => api::skill_descriptor::provider::Type::Codex(()),
+            SkillProvider::Cursor => api::skill_descriptor::provider::Type::Cursor(()),
+            SkillProvider::Gemini => api::skill_descriptor::provider::Type::Gemini(()),
+            SkillProvider::Copilot => api::skill_descriptor::provider::Type::Copilot(()),
+            SkillProvider::Droid => api::skill_descriptor::provider::Type::Droid(()),
+            SkillProvider::Github => api::skill_descriptor::provider::Type::Github(()),
+            SkillProvider::OpenCode => api::skill_descriptor::provider::Type::OpenCode(()),
+        };
+
+        api::skill_descriptor::Provider {
+            r#type: Some(provider_type),
+        }
+    }
+}
+
+impl TryFrom<api::Skill> for ParsedSkill {
+    type Error = SkillConversionError;
+
+    fn try_from(api_skill: api::Skill) -> Result<Self, Self::Error> {
+        let Some(descriptor) = api_skill.descriptor else {
+            return Err(SkillConversionError::MissingDescriptor);
+        };
+        let Some(file_content) = api_skill.content else {
+            return Err(SkillConversionError::MissingContent);
+        };
+        let Some(skill_reference) = descriptor.skill_reference else {
+            return Err(SkillConversionError::MissingReference);
+        };
+        // TODO(pei): Once we refactor ParsedSkill to use SkillDescriptor,
+        // we can pass forward the reference directly to ParsedSkill
+        let path = match skill_reference {
+            api::skill_descriptor::SkillReference::Path(path) => path,
+            _ => "".to_string(), // This is ok only because we don't use the path
+        };
+
+        let Some(Ok(scope)) = descriptor.scope.map(convert_scope) else {
+            return Err(SkillConversionError::ScopeInvalid);
+        };
+
+        let Some(Ok(provider)) = descriptor.provider.map(convert_provider) else {
+            return Err(SkillConversionError::ProviderInvalid);
+        };
+
+        let context: FileContext = file_content.into();
+        let AnyFileContent::StringContent(content) = context.content else {
+            return Err(SkillConversionError::ContentInvalid);
+        };
+
+        let line_range = context.line_range.as_ref();
+
+        Ok(ParsedSkill {
+            path: PathBuf::from(&path),
+            name: descriptor.name,
+            description: descriptor.description,
+            content,
+            line_range: line_range.cloned(),
+            scope,
+            provider,
+        })
+    }
+}
+
+fn convert_scope(scope: api::skill_descriptor::Scope) -> Result<SkillScope, SkillConversionError> {
+    let Some(scope_type) = scope.r#type else {
+        return Err(SkillConversionError::ScopeInvalid);
+    };
+
+    match scope_type {
+        api::skill_descriptor::scope::Type::Home(_) => Ok(SkillScope::Home),
+        api::skill_descriptor::scope::Type::Project(_) => Ok(SkillScope::Project),
+        api::skill_descriptor::scope::Type::Bundled(_) => Ok(SkillScope::Bundled),
+    }
+}
+
+fn convert_provider(
+    provider: api::skill_descriptor::Provider,
+) -> Result<SkillProvider, SkillConversionError> {
+    let Some(provider_type) = provider.r#type else {
+        return Err(SkillConversionError::ProviderInvalid);
+    };
+
+    match provider_type {
+        api::skill_descriptor::provider::Type::Warp(_) => Ok(SkillProvider::Warp),
+        api::skill_descriptor::provider::Type::Agents(_) => Ok(SkillProvider::Agents),
+        api::skill_descriptor::provider::Type::Claude(_) => Ok(SkillProvider::Claude),
+        api::skill_descriptor::provider::Type::Codex(_) => Ok(SkillProvider::Codex),
+        api::skill_descriptor::provider::Type::Cursor(_) => Ok(SkillProvider::Cursor),
+        api::skill_descriptor::provider::Type::Gemini(_) => Ok(SkillProvider::Gemini),
+        api::skill_descriptor::provider::Type::Copilot(_) => Ok(SkillProvider::Copilot),
+        api::skill_descriptor::provider::Type::Droid(_) => Ok(SkillProvider::Droid),
+        api::skill_descriptor::provider::Type::Github(_) => Ok(SkillProvider::Github),
+        api::skill_descriptor::provider::Type::OpenCode(_) => Ok(SkillProvider::OpenCode),
+    }
+}

crates/ai/src/skills/mod.rs

@@ -0,0 +1,14 @@
+mod conversion;
+mod parse_skill;
+mod parser;
+mod read_skills;
+mod skill_provider;
+mod skill_reference;
+
+pub use parse_skill::{parse_bundled_skill, parse_skill, ParsedSkill};
+pub use read_skills::read_skills;
+pub use skill_provider::{
+    get_provider_for_path, home_skills_path, provider_rank, SkillProvider, SkillProviderDefinition,
+    SkillScope, SKILL_PROVIDER_DEFINITIONS,
+};
+pub use skill_reference::SkillReference;

crates/ai/src/skills/parse_skill.rs

@@ -0,0 +1,206 @@
+use anyhow::Result;
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::fmt::Display;
+use std::ops::Range;
+use std::path::{Path, PathBuf};
+
+use super::parser::parse_markdown_file;
+use super::skill_provider::{get_provider_for_path, get_scope_for_path, SkillProvider, SkillScope};
+use thiserror::Error;
+
+const MAX_SKILL_DESCRIPTION_CHARS: usize = 512;
+
+lazy_static! {
+    static ref BLOCK_SEPARATOR: Regex =
+        Regex::new(r"\n\s*\n").expect("Block separator regex should be valid");
+    static ref INCOMPLETE_SENTENCE: Regex =
+        Regex::new(r"[^.!?]*$").expect("Incomplete sentence regex should be valid");
+}
+
+#[derive(Error, Debug)]
+pub enum ParseSkillError {
+    /// This should never happen in practice since we would never read the skill
+    /// file to begin with if the path didn't have a valid parent directory.
+    #[error("Could not derive skill name from path")]
+    CouldNotDeriveSkillNameFromPath,
+}
+
+/// Represents a parsed skill with validated fields
+#[derive(Debug, Clone, PartialEq)]
+pub struct ParsedSkill {
+    pub path: PathBuf,
+    pub name: String,
+    pub description: String,
+    /// The entire content of the file (including front matter)
+    pub content: String,
+    /// The line range where the markdown content (without front matter) is located (1-indexed)
+    /// None if there is no front matter (content is the entire file)
+    pub line_range: Option<Range<usize>>,
+    /// The provider of the skill (Agents, Claude, Codex, or Warp), determined from the path.
+    pub provider: SkillProvider,
+    /// The scope of the skill (home directory vs project directory).
+    pub scope: SkillScope,
+}
+
+impl ParsedSkill {
+    /// Returns true if this skill is bundled with Warp (not a user-editable file).
+    pub fn is_bundled(&self) -> bool {
+        self.scope == SkillScope::Bundled
+    }
+}
+
+impl Display for ParsedSkill {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Skill: {}", self.path.display())
+    }
+}
+
+/// Parse a skill markdown file and validate required fields
+///
+/// # Arguments
+/// * `path` - Path to the skill markdown file to parse
+///
+/// # Returns
+/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
+pub fn parse_skill(path: &Path) -> Result<ParsedSkill> {
+    let provider = get_provider_for_path(path).unwrap_or(SkillProvider::Agents);
+    let scope = get_scope_for_path(path);
+    parse_skill_internal(path, provider, scope)
+}
+
+/// Parse a bundled skill markdown file.
+///
+/// Unlike `parse_skill`, this function does not require the path to match a known
+/// skill provider directory. Bundled skills are always assigned `SkillProvider::Warp`
+/// and `SkillScope::Bundled`.
+///
+/// # Arguments
+/// * `path` - Path to the skill markdown file to parse
+///
+/// # Returns
+/// * `Result<ParsedSkill>` - Parsed skill with validated name and description
+pub fn parse_bundled_skill(path: &Path) -> Result<ParsedSkill> {
+    parse_skill_internal(path, SkillProvider::Warp, SkillScope::Bundled)
+}
+
+fn parse_skill_internal(
+    path: &Path,
+    provider: SkillProvider,
+    scope: SkillScope,
+) -> Result<ParsedSkill> {
+    let parsed = parse_markdown_file(path)?;
+
+    let name = match parsed
+        .front_matter
+        .get("name")
+        .map(|value| value.trim())
+        .filter(|value| !value.is_empty())
+    {
+        Some(name) => name.to_string(),
+        None => derive_skill_name_from_path(path)?,
+    };
+
+    let description = match parsed
+        .front_matter
+        .get("description")
+        .map(|value| value.trim())
+        .filter(|value| !value.is_empty())
+    {
+        Some(description) => description.to_string(),
+        None => truncate_skill_description(
+            &derive_description_from_content(&parsed.content, parsed.line_range.as_ref())
+                .unwrap_or_default(),
+        ),
+    };
+
+    Ok(ParsedSkill {
+        path: path.to_path_buf(),
+        name,
+        description,
+        content: parsed.content,
+        line_range: parsed.line_range,
+        provider,
+        scope,
+    })
+}
+
+fn derive_skill_name_from_path(path: &Path) -> Result<String> {
+    path.parent()
+        .and_then(|parent| parent.file_name())
+        .and_then(|name| name.to_str())
+        .map(|name| name.to_string())
+        .ok_or(ParseSkillError::CouldNotDeriveSkillNameFromPath.into())
+}
+
+fn derive_description_from_content(
+    content: &str,
+    line_range: Option<&Range<usize>>,
+) -> Option<String> {
+    first_paragraph_from_markdown(&extract_markdown_body(content, line_range))
+}
+
+fn extract_markdown_body(content: &str, line_range: Option<&Range<usize>>) -> String {
+    let Some(line_range) = line_range else {
+        return content.to_string();
+    };
+
+    let start = line_range.start.saturating_sub(1);
+    let end = line_range.end.saturating_sub(1);
+    let lines: Vec<&str> = content.lines().collect();
+    if start >= lines.len() {
+        return String::new();
+    }
+
+    let end = end.min(lines.len());
+    lines[start..end].join("\n")
+}
+
+fn first_paragraph_from_markdown(markdown: &str) -> Option<String> {
+    for block in BLOCK_SEPARATOR.split(markdown) {
+        let paragraph: String = block
+            .lines()
+            .map(|line| line.trim())
+            .filter(|line| !line.is_empty() && !line.starts_with('#'))
+            .collect::<Vec<_>>()
+            .join(" ");
+        let paragraph = paragraph.trim();
+        if !paragraph.is_empty() {
+            return Some(paragraph.to_string());
+        }
+    }
+    None
+}
+
+fn truncate_skill_description(description: &str) -> String {
+    let description = description.trim();
+    if description.is_empty() {
+        return String::new();
+    }
+
+    let chars: Vec<char> = description.chars().collect();
+    if chars.len() <= MAX_SKILL_DESCRIPTION_CHARS {
+        return description.to_string();
+    }
+
+    let truncated: String = chars[..MAX_SKILL_DESCRIPTION_CHARS].iter().collect();
+
+    // Drop the trailing incomplete sentence using regex
+    let at_sentence = INCOMPLETE_SENTENCE
+        .replace(&truncated, "")
+        .trim()
+        .to_string();
+    if !at_sentence.is_empty() {
+        return at_sentence;
+    }
+
+    // No sentence boundary found — fall back to word boundary
+    truncated
+        .rfind(char::is_whitespace)
+        .map(|pos| truncated[..pos].trim().to_string())
+        .unwrap_or(truncated)
+}
+
+#[cfg(test)]
+#[path = "parse_skill_test.rs"]
+mod parse_skill_test;

crates/ai/src/skills/parse_skill_test.rs

@@ -0,0 +1,248 @@
+use std::path::PathBuf;
+use tempfile::TempDir;
+
+use super::*;
+
+/// Creates a temporary skill file in a .agents/skills directory
+fn create_temp_skill_file(content: &str) -> (TempDir, PathBuf) {
+    let temp_dir = TempDir::new().unwrap();
+    let skill_dir = temp_dir.path().join(".agents/skills/test-skill");
+    std::fs::create_dir_all(&skill_dir).unwrap();
+    let skill_file = skill_dir.join("SKILL.md");
+    std::fs::write(&skill_file, content).unwrap();
+    (temp_dir, skill_file)
+}
+
+#[test]
+fn test_parse_with_front_matter() {
+    let content = r#"---
+name: your-skill-name
+description: Brief description of what this Skill does and when to use it
+---
+
+# Your Skill Name
+
+## Instructions
+Provide clear, step-by-step guidance for Claude.
+
+## Examples
+Show concrete examples of using this Skill.
+"#;
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.name, "your-skill-name");
+    assert_eq!(
+        result.description,
+        "Brief description of what this Skill does and when to use it"
+    );
+    // Content should include both front matter and markdown
+    assert!(result.content.contains("# Your Skill Name"));
+    assert!(result.content.contains("## Instructions"));
+    assert!(result.content.contains("## Examples"));
+    assert!(result.content.contains("---"));
+    assert!(result.content.contains("name: your-skill-name"));
+    // Verify line_range is set (1-indexed)
+    // Front matter is lines 1-4, markdown content starts at line 5
+    // Total of 12 lines, so line_range is 5..13
+    assert_eq!(result.line_range, Some(5..13));
+    // Verify path is the full file path
+    assert_eq!(result.path, skill_file);
+}
+
+#[test]
+fn test_parse_missing_name_falls_back_to_directory_name() {
+    let content = r#"---
+description: Some description
+---
+
+# Content
+
+This paragraph is for body parsing.
+"#;
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.name, "test-skill");
+    assert_eq!(result.description, "Some description");
+}
+
+#[test]
+fn test_parse_missing_description_falls_back_to_first_paragraph() {
+    let content = r#"---
+name: some-skill
+---
+
+# Heading
+
+This is the first paragraph.
+Still first paragraph.
+
+Second paragraph.
+"#;
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.name, "some-skill");
+    assert_eq!(
+        result.description,
+        "This is the first paragraph. Still first paragraph."
+    );
+}
+
+#[test]
+fn test_parse_missing_both_name_and_description_falls_back() {
+    let content = "---\n\n---\n\n# Heading\n\nThis is the first paragraph.\nStill first paragraph.\n\nSecond paragraph.\n";
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.name, "test-skill");
+    assert_eq!(
+        result.description,
+        "This is the first paragraph. Still first paragraph."
+    );
+}
+
+#[test]
+fn test_parse_no_front_matter_falls_back_to_derived_values() {
+    let content = r#"# Just Content
+
+This is just markdown content without any front matter.
+"#;
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.name, "test-skill");
+    assert_eq!(
+        result.description,
+        "This is just markdown content without any front matter."
+    );
+    assert!(result.line_range.is_none());
+}
+
+#[test]
+fn test_parse_no_skill_provider_defaults_to_agents() {
+    let content = r#"---
+name: some-skill
+description: Some description
+---
+
+# Content
+"#;
+
+    // Create a temp file without a skill provider directory structure
+    let temp_dir = TempDir::new().unwrap();
+    let invalid_file = temp_dir.path().join("invalid.md");
+    std::fs::write(&invalid_file, content).unwrap();
+
+    let result = parse_skill(&invalid_file).unwrap();
+
+    assert_eq!(result.name, "some-skill");
+    assert_eq!(result.description, "Some description");
+    assert_eq!(result.provider, SkillProvider::Agents);
+}
+
+#[test]
+fn test_parse_truncates_long_fallback_description_at_sentence_boundary() {
+    let first_sentence = format!("{}.", "a".repeat(450));
+    let content = format!(
+        r#"---
+name: some-skill
+---
+
+{} {}
+"#,
+        first_sentence,
+        "b".repeat(200)
+    );
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.description, first_sentence);
+}
+
+#[test]
+fn test_parse_truncates_long_fallback_description_at_word_boundary() {
+    let content = format!(
+        r#"---
+name: some-skill
+---
+
+{}
+"#,
+        "word ".repeat(200)
+    );
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert!(result.description.chars().count() <= MAX_SKILL_DESCRIPTION_CHARS);
+    assert!(!result.description.ends_with(' '));
+}
+
+#[test]
+fn test_parse_truncates_fallback_description_with_hard_cut() {
+    let content = format!(
+        "---\nname: some-skill\n---\n\n{}",
+        "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 100)
+    );
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(
+        result.description.chars().count(),
+        MAX_SKILL_DESCRIPTION_CHARS
+    );
+}
+
+#[test]
+fn test_parse_does_not_truncate_user_provided_description() {
+    let description = format!("{} {}", "a".repeat(450), "b".repeat(200));
+    let content = format!(
+        r#"---
+name: some-skill
+description: "{}"
+---
+
+# Content
+"#,
+        description
+    );
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.description, description);
+}
+
+#[test]
+fn test_truncation_does_not_cut_mid_word_like_filename() {
+    // "abc.def" has no sentence boundary (no whitespace after punctuation),
+    // so truncation should fall back to word boundary or hard cut.
+    let long_word = "abc.def".repeat(100);
+    let content = format!("---\nname: some-skill\n---\n\n{long_word}");
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert!(result.description.chars().count() <= MAX_SKILL_DESCRIPTION_CHARS);
+}
+
+#[test]
+fn test_truncation_cuts_at_sentence_boundary() {
+    let first_sentence = "This is a sentence.";
+    let second_sentence_start = " ".to_string() + &"b".repeat(600);
+    let content = format!("---\nname: some-skill\n---\n\n{first_sentence}{second_sentence_start}");
+
+    let (_temp_dir, skill_file) = create_temp_skill_file(&content);
+    let result = parse_skill(&skill_file).unwrap();
+
+    assert_eq!(result.description, "This is a sentence.");
+}

crates/ai/src/skills/parser.rs

@@ -0,0 +1,101 @@
+use anyhow::{Context, Result};
+use regex::Regex;
+use serde_yaml::Value;
+use std::collections::HashMap;
+use std::fs;
+use std::ops::Range;
+use std::path::Path;
+
+/// Represents a parsed markdown file with YAML front matter
+#[derive(Debug)]
+#[allow(dead_code)]
+pub struct ParsedMarkdown {
+    /// The YAML front matter parsed as a map
+    /// For Skills, the front matter is always a single-level map with string keys and string values
+    pub front_matter: HashMap<String, String>,
+    /// The entire content of the file
+    pub content: String,
+    /// The line range where the markdown content (without front matter) is located (1-indexed)
+    /// None if there is no front matter (content is the entire file)
+    pub line_range: Option<Range<usize>>,
+}
+
+/// Parse a markdown file with YAML front matter
+///
+/// # Arguments
+/// * `path` - Path to the markdown file to parse
+///
+/// # Returns
+/// * `Result<ParsedMarkdown>` - Parsed document with front matter and content
+#[allow(dead_code)]
+pub fn parse_markdown_file(path: &Path) -> Result<ParsedMarkdown> {
+    let content = fs::read_to_string(path)?;
+    parse_markdown_content(&content)
+}
+
+/// Parse markdown content with YAML front matter
+#[allow(dead_code)]
+pub(crate) fn parse_markdown_content(content: &str) -> Result<ParsedMarkdown> {
+    // Regex to match YAML front matter at the start of the file
+    // Handles both LF (\n) and CRLF (\r\n) line endings
+    // Allows leading whitespace (spaces, tabs, newlines) before the opening ---
+    // Allows trailing spaces/tabs after --- markers
+    // Pattern: (optional whitespace) --- (optional spaces/tabs) (line ending) (content) (line ending) --- (optional spaces/tabs) (line ending)
+    let front_matter_regex =
+        Regex::new(r"(?ms)\A\s*---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n").unwrap();
+    let captures = front_matter_regex.captures(content);
+
+    if let Some(captures) = captures {
+        // Extract the YAML section (first capture group) and trim to handle extra blank lines
+        let yaml_str = captures.get(1).unwrap().as_str().trim();
+
+        // Parse the YAML into a map (empty front matter is valid — just yields no keys)
+        let front_matter = if yaml_str.is_empty() {
+            HashMap::new()
+        } else {
+            let yaml_value: Value =
+                serde_yaml::from_str(yaml_str).context("Failed to parse YAML front matter")?;
+            match yaml_value {
+                Value::Mapping(map) => map
+                    .iter()
+                    .filter_map(|(key, value)| {
+                        if let Value::String(key_str) = key {
+                            if let Value::String(value_str) = value {
+                                return Some((key_str.clone(), value_str.clone()));
+                            }
+                        }
+
+                        None
+                    })
+                    .collect(),
+                _ => HashMap::new(),
+            }
+        };
+
+        // Get the content after the front matter
+        let content_start = captures.get(0).unwrap().end();
+
+        // Calculate line range for the markdown content (without front matter)
+        // Line numbers are 1-indexed, so we add 1
+        let lines_before_content = content[..content_start].lines().count();
+        let total_lines = content.lines().count();
+        let line_range = Some((lines_before_content + 1)..(total_lines + 1));
+
+        Ok(ParsedMarkdown {
+            front_matter,
+            content: content.to_string(),
+            line_range,
+        })
+    } else {
+        // No front matter found - content is the entire file, so line_range is None
+        Ok(ParsedMarkdown {
+            front_matter: HashMap::new(),
+            content: content.to_string(),
+            line_range: None,
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "parser_test.rs"]
+mod parser_test;

crates/ai/src/skills/parser_test.rs

@@ -0,0 +1,203 @@
+use super::*;
+
+#[test]
+fn test_parse_without_front_matter() {
+    let content = r#"# Hello World
+
+This is just markdown content without front matter.
+"#;
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert!(result.front_matter.is_empty());
+    assert_eq!(result.content, content);
+    // When there's no front matter, line_range should be None
+    assert_eq!(result.line_range, None);
+}
+
+#[test]
+fn test_parse_empty_front_matter() {
+    let content = r#"---
+---
+
+# Content
+"#;
+
+    let result = parse_markdown_content(content).unwrap();
+
+    // Empty front matter (---\n---) doesn't match the regex, so it's treated as no front matter
+    assert!(result.front_matter.is_empty());
+    assert_eq!(result.content, content);
+}
+
+#[test]
+fn test_parse_with_crlf_line_endings() {
+    let content =
+        "---\r\nname: test-skill\r\ndescription: Test description\r\n---\r\n\r\n# Content\r\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "Test description"
+    );
+    // Content should include the entire file (including front matter)
+    assert_eq!(result.content, content);
+    assert!(result.content.contains("# Content"));
+    assert!(result.content.contains("name: test-skill"));
+    // Line range should represent the markdown content after front matter (1-indexed)
+    // Front matter is 4 lines (1-4), markdown content starts at line 5
+    assert_eq!(result.line_range, Some(5..7));
+}
+
+#[test]
+fn test_parse_with_trailing_spaces_after_delimiters() {
+    let content =
+        "---   \nname: test-skill\ndescription: Test description\n---  \t \n\n# Content\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "Test description"
+    );
+}
+
+#[test]
+fn test_parse_with_extra_blank_lines_in_front_matter() {
+    let content = r#"---
+
+name: test-skill
+description: Test description
+
+---
+
+# Content
+"#;
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "Test description"
+    );
+    assert!(result.content.contains("# Content"));
+}
+
+#[test]
+fn test_parse_with_mixed_crlf_and_extra_whitespace() {
+    let content = "---  \r\n\r\nname: test-skill\r\ndescription: Test description\r\n\r\n---\t\r\n\r\n# Content\r\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "Test description"
+    );
+}
+
+#[test]
+fn test_parse_with_tabs_and_spaces() {
+    let content =
+        "---\t  \t\nname: test-skill\ndescription: Test description\n--- \t\n\n# Content\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+}
+
+#[test]
+fn test_crlf_without_proper_front_matter() {
+    let content = "# Hello World\r\n\r\nThis is just markdown content.\r\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert!(result.front_matter.is_empty());
+    assert_eq!(result.content, content);
+}
+
+#[test]
+fn test_parse_with_leading_whitespace_before_front_matter() {
+    let content = r#"
+
+---
+name: test-skill
+description: Test description
+---
+
+# Content
+"#;
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "Test description"
+    );
+
+    // Line numbers (1-indexed):
+    // Line 1: (empty from raw string start)
+    // Line 2: (empty)
+    // Line 3: ---
+    // Line 4: name: test-skill
+    // Line 5: description: Test description
+    // Line 6: ---
+    // Line 7: (empty)
+    // Line 8: # Content
+    assert_eq!(result.line_range, Some(7..9));
+}
+
+#[test]
+fn test_parse_with_spaces_and_newlines_before_front_matter() {
+    let content =
+        "  \n\t\n---\nname: test-skill\ndescription: Test description\n---\n\n# Content\n";
+
+    let result = parse_markdown_content(content).unwrap();
+
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "test-skill");
+}
+
+#[test]
+fn test_content_includes_front_matter_and_line_range() {
+    let content = r#"---
+name: my-skill
+description: My skill description
+---
+
+# My Skill
+
+This is the skill content.
+"#;
+
+    let result = parse_markdown_content(content).unwrap();
+
+    // Verify front matter is parsed correctly
+    assert_eq!(result.front_matter.len(), 2);
+    assert_eq!(result.front_matter.get("name").unwrap(), "my-skill");
+    assert_eq!(
+        result.front_matter.get("description").unwrap(),
+        "My skill description"
+    );
+
+    // Verify content includes the entire file (front matter + markdown)
+    assert_eq!(result.content, content);
+    assert!(result.content.contains("---"));
+    assert!(result.content.contains("name: my-skill"));
+    assert!(result.content.contains("# My Skill"));
+
+    // Verify line_range points to just the markdown content (after front matter, 1-indexed)
+    // Front matter is lines 1-4, markdown content starts at line 5
+    assert_eq!(result.line_range, Some(5..9));
+}

crates/ai/src/skills/read_skills.rs

@@ -0,0 +1,50 @@
+use std::fs;
+use std::path::Path;
+
+use super::parse_skill::{parse_skill, ParsedSkill};
+
+/// Read all skills from a directory containing skill subdirectories
+///
+/// # Arguments
+/// * `path` - The path to a skills directory, e.g. `.claude/skills`
+///
+/// # Returns
+/// * `Vec<ParsedSkill>` - List of successfully parsed skills (invalid files and errors are silently ignored)
+pub fn read_skills(path: &Path) -> Vec<ParsedSkill> {
+    let mut skills = Vec::new();
+
+    // Read all entries in the directory, return empty vec on error
+    let Ok(entries) = fs::read_dir(path) else {
+        return skills;
+    };
+
+    for entry in entries {
+        // Skip entries that fail to read
+        let Ok(entry) = entry else {
+            continue;
+        };
+
+        let entry_path = entry.path();
+
+        // Only process directories
+        if !entry_path.is_dir() {
+            continue;
+        }
+
+        // Look for SKILL.md file in the subdirectory
+        let skill_file_path = entry_path.join("SKILL.md");
+
+        if skill_file_path.exists() {
+            // Attempt to parse the skill file, ignoring errors
+            if let Ok(parsed_skill) = parse_skill(&skill_file_path) {
+                skills.push(parsed_skill);
+            }
+        }
+    }
+
+    skills
+}
+
+#[cfg(test)]
+#[path = "read_skills_test.rs"]
+mod read_skills_test;

crates/ai/src/skills/read_skills_test.rs

@@ -0,0 +1,221 @@
+use super::*;
+use std::fs;
+use tempfile::tempdir;
+
+#[test]
+fn test_read_skills_with_valid_skills() {
+    let temp_dir = tempdir().unwrap();
+    // Create .agents/skills directory structure so skills can have a valid provider
+    let skills_dir = temp_dir.path().join(".agents/skills");
+    fs::create_dir_all(&skills_dir).unwrap();
+
+    // Create first skill directory with valid SKILL.md
+    let skill1_dir = skills_dir.join("skill1");
+    fs::create_dir(&skill1_dir).unwrap();
+    fs::write(
+        skill1_dir.join("SKILL.md"),
+        r#"---
+name: test-skill-1
+description: First test skill
+---
+
+# Test Skill 1
+This is the first test skill.
+"#,
+    )
+    .unwrap();
+
+    // Create second skill directory with valid SKILL.md
+    let skill2_dir = skills_dir.join("skill2");
+    fs::create_dir(&skill2_dir).unwrap();
+    fs::write(
+        skill2_dir.join("SKILL.md"),
+        r#"---
+name: test-skill-2
+description: Second test skill
+---
+
+# Test Skill 2
+This is the second test skill.
+"#,
+    )
+    .unwrap();
+
+    let skills = read_skills(&skills_dir);
+
+    assert_eq!(skills.len(), 2);
+
+    // Find each skill by name
+    let skill1 = skills.iter().find(|s| s.name == "test-skill-1").unwrap();
+    assert_eq!(
+        skill1.path,
+        skill1_dir.join("SKILL.md").to_string_lossy().to_string()
+    );
+    assert_eq!(skill1.description, "First test skill");
+    assert!(skill1.content.contains("# Test Skill 1"));
+    assert!(skill1.content.contains("---"));
+    assert!(skill1.content.contains("name: test-skill-1"));
+    assert_eq!(skill1.line_range, Some(5..8)); // Front matter is lines 1-4, markdown starts at line 5
+
+    let skill2 = skills.iter().find(|s| s.name == "test-skill-2").unwrap();
+    assert_eq!(
+        skill2.path,
+        skill2_dir.join("SKILL.md").to_string_lossy().to_string()
+    );
+    assert_eq!(skill2.description, "Second test skill");
+    assert!(skill2.content.contains("# Test Skill 2"));
+    assert!(skill2.content.contains("---"));
+    assert!(skill2.content.contains("name: test-skill-2"));
+    assert_eq!(skill2.line_range, Some(5..8)); // Front matter is lines 1-4, markdown starts at line 5
+}
+
+#[test]
+fn test_read_skills_ignores_only_truly_invalid_files() {
+    let temp_dir = tempdir().unwrap();
+    // Create .agents/skills directory structure so skills can have a valid provider
+    let skills_dir = temp_dir.path().join(".agents/skills");
+    fs::create_dir_all(&skills_dir).unwrap();
+
+    // Create valid skill
+    let valid_skill_dir = skills_dir.join("valid-skill");
+    fs::create_dir(&valid_skill_dir).unwrap();
+    fs::write(
+        valid_skill_dir.join("SKILL.md"),
+        r#"---
+name: valid-skill
+description: Valid skill
+---
+
+# Valid Skill
+"#,
+    )
+    .unwrap();
+
+    // Create skill missing name (now valid via directory-name fallback)
+    let invalid_skill_dir = skills_dir.join("invalid-skill");
+    fs::create_dir(&invalid_skill_dir).unwrap();
+    fs::write(
+        invalid_skill_dir.join("SKILL.md"),
+        r#"---
+description: Invalid skill missing name
+---
+
+# Invalid Skill
+"#,
+    )
+    .unwrap();
+
+    // Create skill with no front matter (now valid via full fallback)
+    let no_frontmatter_dir = skills_dir.join("no-frontmatter-skill");
+    fs::create_dir(&no_frontmatter_dir).unwrap();
+    fs::write(
+        no_frontmatter_dir.join("SKILL.md"),
+        r#"# No Front Matter Skill
+
+No front matter here.
+"#,
+    )
+    .unwrap();
+
+    let skills = read_skills(&skills_dir);
+
+    // All three skills should be returned — none are truly invalid
+    assert_eq!(skills.len(), 3);
+
+    let valid_skill = skills.iter().find(|s| s.name == "valid-skill").unwrap();
+    assert_eq!(valid_skill.description, "Valid skill");
+    assert!(valid_skill.content.contains("---"));
+    assert_eq!(valid_skill.line_range, Some(5..7));
+
+    let fallback_name_skill = skills.iter().find(|s| s.name == "invalid-skill").unwrap();
+    assert_eq!(
+        fallback_name_skill.description,
+        "Invalid skill missing name"
+    );
+    assert!(fallback_name_skill.content.contains("# Invalid Skill"));
+
+    let no_fm_skill = skills
+        .iter()
+        .find(|s| s.name == "no-frontmatter-skill")
+        .unwrap();
+    assert_eq!(no_fm_skill.description, "No front matter here.");
+    assert!(no_fm_skill.line_range.is_none());
+}
+
+#[test]
+fn test_read_skills_empty_directory() {
+    let temp_dir = tempdir().unwrap();
+    let skills_dir = temp_dir.path();
+
+    let skills = read_skills(skills_dir);
+
+    assert_eq!(skills.len(), 0);
+}
+
+#[test]
+fn test_read_skills_no_skill_files() {
+    let temp_dir = tempdir().unwrap();
+    let skills_dir = temp_dir.path();
+
+    // Create directories without SKILL.md files
+    let dir1 = skills_dir.join("dir1");
+    fs::create_dir(&dir1).unwrap();
+
+    let dir2 = skills_dir.join("dir2");
+    fs::create_dir(&dir2).unwrap();
+    fs::write(dir2.join("README.md"), "Not a skill file").unwrap();
+
+    let skills = read_skills(skills_dir);
+
+    assert_eq!(skills.len(), 0);
+}
+
+#[test]
+fn test_read_skills_ignores_files_in_root() {
+    let temp_dir = tempdir().unwrap();
+    // Create .agents/skills directory structure so skills can have a valid provider
+    let skills_dir = temp_dir.path().join(".agents/skills");
+    fs::create_dir_all(&skills_dir).unwrap();
+
+    // Create a valid skill in a subdirectory
+    let skill_dir = skills_dir.join("valid-skill");
+    fs::create_dir(&skill_dir).unwrap();
+    fs::write(
+        skill_dir.join("SKILL.md"),
+        r#"---
+name: valid-skill
+description: Valid skill in subdirectory
+---
+
+# Valid Skill
+"#,
+    )
+    .unwrap();
+
+    // Create a SKILL.md file in the root directory (should be ignored)
+    fs::write(
+        skills_dir.join("SKILL.md"),
+        r#"---
+name: root-skill
+description: This should be ignored
+---
+
+# Root Skill
+"#,
+    )
+    .unwrap();
+
+    let skills = read_skills(&skills_dir);
+
+    // Only the skill in the subdirectory should be returned
+    assert_eq!(skills.len(), 1);
+    assert_eq!(skills[0].name, "valid-skill");
+    assert_eq!(skills[0].line_range, Some(5..7));
+}
+
+#[test]
+fn test_read_skills_nonexistent_directory() {
+    let skills = read_skills(Path::new("/nonexistent/path/that/does/not/exist"));
+
+    assert_eq!(skills.len(), 0);
+}

crates/ai/src/skills/skill_provider.rs

@@ -0,0 +1,238 @@
+//! Skill provider definitions and utilities.
+//!
+//! This module defines the supported skill providers (i.e. Agents, Claude, Codex, Warp) and their
+//! associated skills directory paths. It provides utilities for looking up providers
+//! from paths and vice versa.
+use dirs::home_dir;
+use std::path::{Path, PathBuf};
+use std::sync::LazyLock;
+
+use serde::{Deserialize, Serialize};
+
+use strum_macros::{Display, EnumString, VariantNames};
+use warp_core::ui::color::CLAUDE_ORANGE;
+use warp_core::ui::icons::Icon;
+use warp_core::ui::theme::Fill;
+
+/// Represents a skill provider/origin (Agents, Claude, Codex, or Warp).
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    Serialize,
+    Deserialize,
+    Display,
+    EnumString,
+    VariantNames,
+)]
+pub enum SkillProvider {
+    Warp,
+    Agents,
+    Claude,
+    Codex,
+    Cursor,
+    Gemini,
+    Copilot,
+    Droid,
+    Github,
+    OpenCode,
+}
+
+/// Represents the scope of a skill (home directory vs project directory).
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    Serialize,
+    Deserialize,
+    Default,
+    Display,
+    EnumString,
+    VariantNames,
+)]
+pub enum SkillScope {
+    /// Skills from the user's home directory (e.g., `~/.agents/skills`).
+    #[default]
+    Home,
+    /// Skills from a project directory (e.g., `./repo/.agents/skills`).
+    Project,
+    /// Bundled skills distributed with Warp.
+    Bundled,
+}
+
+/// Definition of a skill provider including its directory path.
+pub struct SkillProviderDefinition {
+    pub provider: SkillProvider,
+    /// Relative path from root (repo or home), constructed with platform-aware joining.
+    pub skills_path: PathBuf,
+}
+
+impl SkillProvider {
+    /// Returns the default icon for this provider.
+    pub fn icon(&self) -> Icon {
+        match self {
+            SkillProvider::Claude => Icon::ClaudeLogo,
+            SkillProvider::Codex => Icon::OpenAILogo,
+            SkillProvider::Gemini => Icon::GeminiLogo,
+            SkillProvider::Droid => Icon::DroidLogo,
+            SkillProvider::OpenCode => Icon::OpenCodeLogo,
+            SkillProvider::Warp
+            | SkillProvider::Agents
+            | SkillProvider::Cursor
+            | SkillProvider::Copilot
+            | SkillProvider::Github => Icon::WarpLogoLight,
+        }
+    }
+
+    /// Returns the icon fill for this provider, using `fallback` for providers that
+    /// don't require a specific color. Claude uses its branded salmon color instead.
+    pub fn icon_fill(&self, fallback: Fill) -> Fill {
+        match self {
+            SkillProvider::Claude => Fill::Solid(CLAUDE_ORANGE),
+            _ => fallback,
+        }
+    }
+}
+
+/// All provider definitions. Order determines precedence (first = highest priority).
+pub static SKILL_PROVIDER_DEFINITIONS: LazyLock<Vec<SkillProviderDefinition>> =
+    LazyLock::new(|| {
+        vec![
+            SkillProviderDefinition {
+                provider: SkillProvider::Agents,
+                skills_path: PathBuf::from(".agents").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Warp,
+                skills_path: PathBuf::from(".warp").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Claude,
+                skills_path: PathBuf::from(".claude").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Codex,
+                skills_path: PathBuf::from(".codex").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Cursor,
+                skills_path: PathBuf::from(".cursor").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Gemini,
+                skills_path: PathBuf::from(".gemini").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Copilot,
+                skills_path: PathBuf::from(".copilot").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Droid,
+                skills_path: PathBuf::from(".factory").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::Github,
+                skills_path: PathBuf::from(".github").join("skills"),
+            },
+            SkillProviderDefinition {
+                provider: SkillProvider::OpenCode,
+                skills_path: PathBuf::from(".opencode").join("skills"),
+            },
+        ]
+    });
+
+/// Returns the precedence rank of a provider based on its position in [`SKILL_PROVIDER_DEFINITIONS`].
+pub fn provider_rank(provider: SkillProvider) -> usize {
+    SKILL_PROVIDER_DEFINITIONS
+        .iter()
+        .position(|def| def.provider == provider)
+        // NOTE: Each SkillProvider should map to a unique SkillProviderDefinition
+        // so we should never reach this path.
+        .unwrap_or(usize::MAX)
+}
+
+pub fn home_skills_path(provider: SkillProvider) -> Option<PathBuf> {
+    if provider == SkillProvider::Warp {
+        return warp_core::paths::warp_home_skills_dir();
+    }
+    let definition = SKILL_PROVIDER_DEFINITIONS
+        .iter()
+        .find(|def| def.provider == provider)?;
+    home_dir().map(|home_dir| home_dir.join(&definition.skills_path))
+}
+
+/// Returns the skill provider for a given path, if it matches a known skill provider directory.
+/// For example:
+///   get_provider_for_path(Path::new("/repo/.claude/skills/my-skill/SKILL.md")) returns Some(SkillProvider::Claude).
+/// Handles both SKILL.md files and files nested within a skill directory.
+pub fn get_provider_for_path(path: &Path) -> Option<SkillProvider> {
+    let path_components: Vec<_> = path.components().collect();
+
+    for def in SKILL_PROVIDER_DEFINITIONS.iter() {
+        if home_skills_path(def.provider)
+            .into_iter()
+            .any(|home_skills_path| path.starts_with(home_skills_path))
+        {
+            return Some(def.provider);
+        }
+
+        // Retrieves path components for the skill provider directory (i.e., [".claude", "skills"])
+        let skill_components: Vec<_> = def.skills_path.components().collect();
+
+        // Checks if some consecutive components of the path match the skill provider directory
+        for window in path_components.windows(skill_components.len()) {
+            if window == skill_components.as_slice() {
+                return Some(def.provider);
+            }
+        }
+    }
+    None
+}
+
+/// Returns the skill scope (Home or Project) for a given path.
+/// A skill is considered a "Home" skill if its path starts with the user's home directory.
+/// Otherwise, it's a "Project" skill.
+pub fn get_scope_for_path(path: &Path) -> SkillScope {
+    for def in SKILL_PROVIDER_DEFINITIONS.iter() {
+        if home_skills_path(def.provider)
+            .into_iter()
+            .any(|home_skills_path| path.starts_with(home_skills_path))
+        {
+            return SkillScope::Home;
+        }
+    }
+    SkillScope::Project
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        get_provider_for_path, get_scope_for_path, home_skills_path, SkillProvider, SkillScope,
+    };
+
+    #[test]
+    fn warp_home_skills_path_uses_warp_home_path() {
+        assert_eq!(
+            home_skills_path(SkillProvider::Warp),
+            warp_core::paths::warp_home_skills_dir()
+        );
+    }
+
+    #[test]
+    fn warp_home_skill_path_is_home_warp_skill() {
+        let Some(warp_home_skills_dir) = warp_core::paths::warp_home_skills_dir() else {
+            eprintln!("Skipping test: home directory not available");
+            return;
+        };
+        let path = warp_home_skills_dir.join("my-skill").join("SKILL.md");
+
+        assert_eq!(get_provider_for_path(&path), Some(SkillProvider::Warp));
+        assert_eq!(get_scope_for_path(&path), SkillScope::Home);
+    }
+}

crates/ai/src/skills/skill_reference.rs

@@ -0,0 +1,35 @@
+use serde::{Deserialize, Serialize};
+use std::{fmt, path::PathBuf};
+
+/// An unique reference to a skill.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
+pub enum SkillReference {
+    /// A skill identified by the path to its SKILL.md file.
+    Path(PathBuf),
+    /// A bundled skill distributed with Warp.
+    BundledSkillId(String),
+}
+
+impl fmt::Display for SkillReference {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SkillReference::Path(path) => path.display().fmt(f),
+            SkillReference::BundledSkillId(id) => write!(f, "@warp-skill:{id}"),
+        }
+    }
+}
+
+impl From<SkillReference> for warp_multi_agent_api::skill_descriptor::SkillReference {
+    fn from(reference: SkillReference) -> Self {
+        match reference {
+            SkillReference::Path(path) => {
+                warp_multi_agent_api::skill_descriptor::SkillReference::Path(
+                    path.to_string_lossy().to_string(),
+                )
+            }
+            SkillReference::BundledSkillId(id) => {
+                warp_multi_agent_api::skill_descriptor::SkillReference::BundledSkillId(id)
+            }
+        }
+    }
+}

crates/ai/src/telemetry.rs

@@ -0,0 +1,179 @@
+use std::time::Duration;
+
+use serde::Serialize;
+use serde_json::{json, Value};
+use strum_macros::{EnumDiscriminants, EnumIter};
+use warp_core::{
+    features::FeatureFlag,
+    register_telemetry_event,
+    telemetry::{EnablementState, TelemetryEvent, TelemetryEventDesc},
+};
+
+#[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
+#[derive(Clone, EnumDiscriminants)]
+#[strum_discriminants(derive(EnumIter))]
+pub enum AITelemetryEvent {
+    MerkleTreeSnapshotRebuildSuccess {
+        duration: Duration,
+    },
+    MerkleTreeSnapshotRebuildFailed {
+        error: String,
+    },
+    MerkleTreeSnapshotDiffSuccess {
+        duration: Duration,
+    },
+    MerkleTreeSnapshotDiffFailed {
+        error: String,
+    },
+    SyncCodebaseContextSuccess {
+        total_sync_duration: Duration,
+        flushed_node_count: usize,
+        flushed_fragment_count: usize,
+        total_fragment_size_bytes: usize,
+        sync_type: CodebaseContextSyncType,
+        cache_population_error: Option<String>,
+    },
+    SyncCodebaseContextFailed {
+        error: String,
+        sync_type: CodebaseContextSyncType,
+    },
+    BuildTreeFailed {
+        error: String,
+    },
+    BuildTreeSuccess {
+        file_traversal_duration: Duration,
+        merkle_tree_parse_duration: Duration,
+    },
+}
+
+#[cfg_attr(not(feature = "local_fs"), allow(dead_code))]
+#[derive(Clone, Serialize)]
+pub enum CodebaseContextSyncType {
+    Full,
+    Initial,
+    Incremental,
+}
+
+impl TelemetryEvent for AITelemetryEvent {
+    fn name(&self) -> &'static str {
+        AITelemetryEventDiscriminants::from(self).name()
+    }
+
+    fn description(&self) -> &'static str {
+        AITelemetryEventDiscriminants::from(self).description()
+    }
+
+    fn enablement_state(&self) -> EnablementState {
+        AITelemetryEventDiscriminants::from(self).enablement_state()
+    }
+
+    fn payload(&self) -> Option<Value> {
+        match self {
+            Self::MerkleTreeSnapshotRebuildSuccess { duration } => Some(json!({
+                "duration": duration,
+            })),
+            Self::MerkleTreeSnapshotRebuildFailed { error } => Some(json!({
+                "error": error,
+            })),
+            Self::MerkleTreeSnapshotDiffSuccess { duration } => Some(json!({
+                "duration": duration,
+            })),
+            Self::MerkleTreeSnapshotDiffFailed { error } => Some(json!({
+                "error": error,
+            })),
+            Self::SyncCodebaseContextSuccess {
+                total_sync_duration,
+                sync_type,
+                flushed_node_count,
+                flushed_fragment_count,
+                total_fragment_size_bytes,
+                cache_population_error,
+            } => Some(json!({
+                "total_sync_duration": total_sync_duration,
+                "sync_type": sync_type,
+                "flushed_node_count": flushed_node_count,
+                "flushed_fragment_count": flushed_fragment_count,
+                "total_fragment_size_bytes": total_fragment_size_bytes,
+                "cache_population_error": cache_population_error
+            })),
+            Self::SyncCodebaseContextFailed { error, sync_type } => Some(json!({
+                "error": error,
+                "sync_type": sync_type
+            })),
+            Self::BuildTreeFailed { error } => Some(json!({
+                "error": error
+            })),
+            Self::BuildTreeSuccess {
+                file_traversal_duration,
+                merkle_tree_parse_duration,
+            } => Some(json!({
+                "file_traversal_duration": file_traversal_duration,
+                "merkle_tree_parse_duration": merkle_tree_parse_duration
+            })),
+        }
+    }
+
+    fn contains_ugc(&self) -> bool {
+        match self {
+            Self::MerkleTreeSnapshotRebuildSuccess { .. }
+            | Self::MerkleTreeSnapshotRebuildFailed { .. }
+            | Self::MerkleTreeSnapshotDiffSuccess { .. }
+            | Self::MerkleTreeSnapshotDiffFailed { .. }
+            | Self::SyncCodebaseContextFailed { .. }
+            | Self::SyncCodebaseContextSuccess { .. }
+            | Self::BuildTreeFailed { .. }
+            | Self::BuildTreeSuccess { .. } => false,
+        }
+    }
+
+    fn event_descs() -> impl Iterator<Item = Box<dyn TelemetryEventDesc>> {
+        warp_core::telemetry::enum_events::<Self>()
+    }
+}
+
+impl TelemetryEventDesc for AITelemetryEventDiscriminants {
+    fn name(&self) -> &'static str {
+        match self {
+            Self::MerkleTreeSnapshotRebuildSuccess => {
+                "AgentMode.MerkleTreeSnapshot.Rebuild.Success"
+            }
+            Self::MerkleTreeSnapshotRebuildFailed => "AgentMode.MerkleTreeSnapshot.Rebuild.Failed",
+            Self::MerkleTreeSnapshotDiffSuccess => "AgentMode.MerkleTreeSnapshot.Diff.Success",
+            Self::MerkleTreeSnapshotDiffFailed => "AgentMode.MerkleTreeSnapshot.Diff.Failed",
+            Self::SyncCodebaseContextSuccess => "AgentMode.SyncCodebaseContext.Success",
+            Self::SyncCodebaseContextFailed => "AgentMode.SyncCodebaseContext.Failed",
+            Self::BuildTreeFailed => "AgentMode.SyncCodebaseContext.BuildTree.Failed",
+            Self::BuildTreeSuccess => "AgentMode.SyncCodebaseContext.BuildTree.Success",
+        }
+    }
+
+    fn description(&self) -> &'static str {
+        match self {
+            Self::MerkleTreeSnapshotRebuildSuccess => {
+                "Successfully rebuilt merkle tree from snapshot"
+            }
+            Self::MerkleTreeSnapshotRebuildFailed => "Failed to rebuild merkle tree from snapshot",
+            Self::MerkleTreeSnapshotDiffSuccess => "Successfully diffed merkle tree snapshot",
+            Self::MerkleTreeSnapshotDiffFailed => "Failed to diff merkle tree snapshot",
+            Self::SyncCodebaseContextSuccess => "Successfully synced codebase context",
+            Self::SyncCodebaseContextFailed => "Failed to sync codebase context",
+            Self::BuildTreeFailed => "Failed to build merkle tree for codebase context",
+            Self::BuildTreeSuccess => "Successfully built merkle tree for codebase context",
+        }
+    }
+
+    fn enablement_state(&self) -> EnablementState {
+        match self {
+            Self::MerkleTreeSnapshotRebuildSuccess
+            | Self::MerkleTreeSnapshotRebuildFailed
+            | Self::MerkleTreeSnapshotDiffSuccess
+            | Self::MerkleTreeSnapshotDiffFailed
+            | Self::SyncCodebaseContextFailed
+            | Self::SyncCodebaseContextSuccess
+            | Self::BuildTreeFailed
+            | Self::BuildTreeSuccess => EnablementState::Flag(FeatureFlag::FullSourceCodeEmbedding),
+        }
+    }
+}
+
+register_telemetry_event!(AITelemetryEvent);

crates/ai/src/workspace.rs

@@ -0,0 +1,94 @@
+use chrono::{DateTime, Days, Utc};
+use std::path::PathBuf;
+
+/// Public-facing metadata persisted in SQLite
+#[derive(Debug, Default, Clone)]
+pub struct WorkspaceMetadata {
+    pub path: PathBuf,
+    pub navigated_ts: Option<DateTime<Utc>>,
+    pub modified_ts: Option<DateTime<Utc>>,
+    pub queried_ts: Option<DateTime<Utc>>,
+}
+
+impl WorkspaceMetadata {
+    /// Surface most recently navigated first
+    pub fn most_recently_navigated(a: &Self, b: &Self) -> std::cmp::Ordering {
+        match (a.navigated_ts, b.navigated_ts) {
+            (Some(a_ts), Some(b_ts)) => b_ts.cmp(&a_ts),
+            (Some(_), None) => std::cmp::Ordering::Less,
+            (None, Some(_)) => std::cmp::Ordering::Greater,
+            (None, None) => a.path.cmp(&b.path),
+        }
+    }
+
+    /// Surface most recently touched first
+    pub fn most_recently_touched(a: &Self, b: &Self) -> std::cmp::Ordering {
+        match (a.last_touched(), b.last_touched()) {
+            (Some(a_ts), Some(b_ts)) => b_ts.cmp(&a_ts),
+            (Some(_), None) => std::cmp::Ordering::Less,
+            (None, Some(_)) => std::cmp::Ordering::Greater,
+            (None, None) => a.path.cmp(&b.path),
+        }
+    }
+
+    /// The most recent time this codebase index was navigated to, queried or modified.
+    pub fn last_touched(&self) -> Option<DateTime<Utc>> {
+        let mut last_access_time: Option<DateTime<Utc>> = None;
+        if let Some(nav_ts) = self.navigated_ts {
+            last_access_time = last_access_time
+                .map(|old_time| old_time.max(nav_ts))
+                .or(Some(nav_ts));
+        }
+        if let Some(mod_ts) = self.modified_ts {
+            last_access_time = last_access_time
+                .map(|old_time| old_time.max(mod_ts))
+                .or(Some(mod_ts));
+        }
+        if let Some(query_ts) = self.queried_ts {
+            last_access_time = last_access_time
+                .map(|old_time| old_time.max(query_ts))
+                .or(Some(query_ts));
+        }
+        last_access_time
+    }
+
+    pub fn is_expired(&self, current_time: DateTime<Utc>, shelf_life_days: u64) -> bool {
+        let Some(last_touch) = self.last_touched() else {
+            return true;
+        };
+        last_touch
+            .checked_add_days(Days::new(shelf_life_days))
+            .unwrap_or_default()
+            < current_time
+    }
+}
+
+/// An event to update the workspace metadata.
+#[derive(Debug, Clone, Copy)]
+pub enum WorkspaceMetadataEvent {
+    Queried,
+    Modified,
+    Created,
+}
+
+impl From<WorkspaceMetadata> for persistence::model::NewWorkspaceMetadata {
+    fn from(value: WorkspaceMetadata) -> Self {
+        Self {
+            repo_path: value.path.to_string_lossy().into_owned(),
+            navigated_ts: value.navigated_ts.map(|utc_dt| utc_dt.naive_utc()),
+            modified_ts: value.modified_ts.map(|utc_dt| utc_dt.naive_utc()),
+            queried_ts: value.queried_ts.map(|utc_dt| utc_dt.naive_utc()),
+        }
+    }
+}
+
+impl From<persistence::model::WorkspaceMetadata> for WorkspaceMetadata {
+    fn from(value: persistence::model::WorkspaceMetadata) -> Self {
+        Self {
+            path: PathBuf::from(value.repo_path),
+            navigated_ts: value.navigated_ts.map(|naive_ts| naive_ts.and_utc()),
+            modified_ts: value.modified_ts.map(|naive_ts| naive_ts.and_utc()),
+            queried_ts: value.queried_ts.map(|naive_ts| naive_ts.and_utc()),
+        }
+    }
+}

crates/app-installation-detection/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "app-installation-detection"
+version = "0.1.0"
+edition = "2021"
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+axum.workspace = true
+axum-extra = "0.10.1"
+command.workspace = true
+tokio = { version = "1.37", features = ["full"] }
+nix.workspace = true
+tower.workspace = true
+tower-http = { workspace = true, features = ["trace", "cors"] }
+tracing.workspace = true
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+warp_cli.workspace = true
+
+[target.'cfg(target_os = "windows")'.dependencies]
+win32job = "2.0.1"

crates/app-installation-detection/src/lib.rs

@@ -0,0 +1,54 @@
+use std::time::Duration;
+
+use axum::body::Body;
+use axum::http::request::Parts;
+use axum::http::{HeaderValue, Method, Response};
+use axum::{extract::Request, routing::get, Router};
+use tower::ServiceBuilder;
+use tower_http::cors::{AllowOrigin, CorsLayer};
+use tower_http::trace::TraceLayer;
+use tracing::Span;
+
+pub fn make_router() -> Router {
+    let trace_service = ServiceBuilder::new().layer(
+        TraceLayer::new_for_http()
+            .make_span_with(|request: &Request<Body>| {
+                tracing::info_span!(
+                    "http-request",
+                    method = request.method().as_str(),
+                    uri = request.uri().to_string(),
+                )
+            })
+            .on_request(())
+            .on_response(
+                |response: &Response<Body>, _latency: Duration, _span: &Span| {
+                    tracing::info!(response_status = response.status().as_u16());
+                },
+            )
+            .on_body_chunk(())
+            .on_eos(())
+            .on_failure(()),
+    );
+
+    // We allow requests from localhost, warp.dev and any subdomain of warp.dev.
+    let allow_origin_predicate =
+        AllowOrigin::predicate(|origin: &HeaderValue, _request_parts: &Parts| {
+            origin == "http://localhost:8080"
+                || origin == "http://localhost:8082"
+                || origin == "https://warp.dev"
+                || origin.as_bytes().ends_with(b".warp.dev")
+        });
+
+    let cors = CorsLayer::new()
+        .allow_methods([Method::GET])
+        .allow_origin(allow_origin_predicate);
+
+    Router::new()
+        .route_service("/install_detection", get(detect_installation))
+        .layer(trace_service)
+        .layer(cors)
+}
+
+async fn detect_installation() -> &'static str {
+    "ok"
+}

crates/asset_cache/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "asset_cache"
+version = "0.0.0"
+edition = "2024"
+authors.workspace = true
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+async-compat.workspace = true
+async-fs.workspace = true
+bytes.workspace = true
+cfg-if.workspace = true
+futures.workspace = true
+log.workspace = true
+reqwest.workspace = true
+url.workspace = true
+warpui_core.workspace = true

crates/asset_cache/src/lib.rs

@@ -0,0 +1,176 @@
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use anyhow::Result;
+use async_fs::{OpenOptions, create_dir_all};
+use bytes::Bytes;
+use futures::AsyncWriteExt;
+use reqwest::Url;
+use warpui_core::assets::asset_cache::{
+    Asset, AssetCache, AssetSource, AssetState, AsyncAssetId, AsyncAssetType,
+};
+
+/// Namespace marker for URL-based async asset sources without persistence.
+pub struct UrlAssetWithoutPersistence;
+impl AsyncAssetType for UrlAssetWithoutPersistence {}
+
+/// Namespace marker for URL-based async asset sources with persistence.
+///
+/// This is intentionally separate from `UrlAssetWithoutPersistence` to allow
+/// ensure we persist the asset even if we fetched it once already without
+/// persistence.
+pub struct UrlAssetWithPersistence;
+impl AsyncAssetType for UrlAssetWithPersistence {}
+
+/// Creates an [`AssetSource::Async`] that fetches bytes from the given URL
+/// without persisting them to the local filesystem.
+pub fn url_source(url: impl Into<String>) -> AssetSource {
+    let url = url.into();
+    let url_for_fetch = url.clone();
+    AssetSource::Async {
+        id: AsyncAssetId::new::<UrlAssetWithoutPersistence>(url),
+        fetch: Arc::new(move || {
+            let url = url_for_fetch.clone();
+            Box::pin(async move {
+                let parsed = Url::parse(&url)?;
+                fetch_file_to_memory(parsed).await
+            })
+        }),
+    }
+}
+
+/// Creates an [`AssetSource::Async`] that fetches bytes from the given URL,
+/// persisting them to a file under `cache_dir` for future reads.
+pub fn url_source_with_persistence(url: impl Into<String>, cache_dir: &Path) -> AssetSource {
+    let url = url.into();
+    let url_for_fetch = url.clone();
+    let cache_dir_owned = cache_dir.to_path_buf();
+    AssetSource::Async {
+        id: AsyncAssetId::new::<UrlAssetWithPersistence>(url),
+        fetch: Arc::new(move || {
+            let url = url_for_fetch.clone();
+            let cache_dir = cache_dir_owned.clone();
+            Box::pin(async move {
+                let parsed = Url::parse(&url)?;
+                let file = get_file_path_for_asset(&parsed, &cache_dir);
+                fetch_asset_from_url(parsed, Some(file)).await
+            })
+        }),
+    }
+}
+
+/// Extension trait that adds URL-based asset loading to [`AssetCache`].
+pub trait AssetCacheExt {
+    /// Loads an asset from a URL, optionally persisting the fetched bytes to
+    /// a file under `cache_dir` for future cache hits.
+    fn load_asset_from_url<T: Asset>(&self, url: &str, cache_dir: Option<&Path>) -> AssetState<T>;
+}
+
+impl AssetCacheExt for AssetCache {
+    fn load_asset_from_url<T: Asset>(&self, url: &str, cache_dir: Option<&Path>) -> AssetState<T> {
+        let source = match cache_dir {
+            Some(dir) => url_source_with_persistence(url, dir),
+            None => url_source(url),
+        };
+        self.load_asset(source)
+    }
+}
+
+/// Fetches a file from the given `url` to memory.
+async fn fetch_file_to_memory(url: Url) -> Result<Bytes, anyhow::Error> {
+    cfg_if::cfg_if! {
+        if #[cfg(target_family = "wasm")] {
+            let response = reqwest::get(url).await?;
+        } else {
+            // On non-web platforms, reqwest expects that it is operating within
+            // a Tokio-compatible runtime, so use async-compat to wrap the call
+            // so reqwest's expectations are met.
+            let response = async_compat::Compat::new(async move { reqwest::get(url).await }).await?;
+        }
+    }
+    let content = response.bytes().await?;
+    Ok(content)
+}
+
+/// Given a url and a directory where cached artifacts are stored, returns a unique
+/// file path for an asset.
+fn get_file_path_for_asset(url: &Url, cache_dir: &Path) -> PathBuf {
+    // Hash the URL so that we can derive a "safe" file name for it. We need something
+    // unique and not too long (most filesystems have a maximum length limit for file
+    // names. On MacOS it's 255).
+    let mut hasher = DefaultHasher::new();
+    url.hash(&mut hasher);
+    let digest = hasher.finish();
+    // Stringify the bytes in hexadecimal. Be careful not to use base64-digests in file
+    // names b/c base64 uses a mix of upper and lowercase chars, which is problematic on
+    // case-insensitive filesystems such as MacOS
+    let filename = format!("{digest:x}");
+    cache_dir.join(filename)
+}
+
+async fn persist_bytes(bytes: &Bytes, file: &Path) {
+    let Some(parent_folder) = file.parent() else {
+        log::error!("attempted to write cache file in filesystem root");
+        return;
+    };
+
+    if let Err(e) = create_dir_all(parent_folder).await {
+        log::error!("Error creating directory for cache files: {e:#}");
+    }
+
+    let mut file = match OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(file)
+        .await
+    {
+        Ok(file) => file,
+        Err(e) => {
+            log::error!("Error opening file: {e:#}");
+            return;
+        }
+    };
+
+    if let Err(e) = file.write_all(bytes).await {
+        log::error!("Error writing to file: {e:#}");
+    }
+
+    if let Err(e) = file.flush().await {
+        log::error!("Error flushing file: {e:#}");
+    };
+}
+
+async fn fetch_file_and_persist_bytes(url: Url, file: Option<PathBuf>) -> Result<Bytes> {
+    let result = fetch_file_to_memory(url).await;
+
+    // If the bytes should be written to a file, do so now.
+    if let Ok(bytes) = result.as_ref()
+        && let Some(filename) = file
+    {
+        persist_bytes(bytes, &filename).await;
+    }
+
+    result
+}
+
+async fn fetch_asset_from_url(url: Url, file: Option<PathBuf>) -> Result<Bytes> {
+    match file {
+        // If a file path is specified and that file path currently exists in the
+        // user's filesystem, read the bytes out of the file.
+        Some(filename) if filename.exists() => {
+            log::debug!("Reading bytes from cached file: {filename:?}");
+            let buffer = async_fs::read(filename.clone()).await?;
+
+            // If buffer is empty, try to fetch from url instead
+            if buffer.is_empty() {
+                return fetch_file_and_persist_bytes(url, Some(filename)).await;
+            }
+
+            Ok(buffer.into())
+        }
+        // Otherwise, fetch the bytes from the url.
+        _ => fetch_file_and_persist_bytes(url, file).await,
+    }
+}

crates/asset_macro/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "asset_macro"
+version = "0.1.0"
+edition = "2021"
+publish.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1"
+syn = { version = "2", default-features = false, features = ["derive", "parsing", "proc-macro", "printing"] }
+proc-macro2 = "1"
+sha2.workspace = true
+warp_util.workspace = true

crates/asset_macro/src/lib.rs

@@ -0,0 +1,157 @@
+//! This module defines a set of macros used to reference assets in Warp.
+//!
+//! The three types of assets are:
+//! - Bundled: These are always included in the app bundle. These files are located in `app/assets/bundled`.
+//!   Access with `bundled_asset!([path of asset relative to app/assets/bundled])`.
+//! - Remote: These are always fetched remotely based on the asset name and a hash of the contents.
+//!   These files are located in `app/assets/remote`. Access with
+//!   `remote_asset!(path of asset relative to app/assets/remote])`.
+//! - Bundled for native builds and remote for web builds: Keeping the size of the web build small
+//!   is critical for having fast load times, so many of the larger assets are split out. These
+//!   files live in `app/assets/async`. Access with
+//!   `bundled_or_fetched!(path of asset relative to app/assets/async])`.
+//!
+//! These macros check for the existence of the asset at the appropriate location before returning
+//! an `AssetSource` with the appropriate bundle reference or URL.
+//!
+//! You can specify a specific folder under `app/assets` to look in as the second argument to any
+//! of these macros, but you probably shouldn't be doing that.
+
+#![recursion_limit = "1024"]
+#[macro_use]
+extern crate quote;
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use sha2::Digest;
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
+use syn::{parse::Parse, Token};
+use syn::{parse_macro_input, LitStr};
+use warp_util::assets::{ASSETS_DIR, ASYNC_ASSETS_DIR, BUNDLED_ASSETS_DIR, REMOTE_ASSETS_DIR};
+
+struct MacroArgs {
+    /// The name of the asset. E.g. `jpg/jellyfish_bg.jpg`
+    asset_name: LitStr,
+    /// The asset subfolder under `app/assets`. E.g. `async`.
+    asset_folder: Option<LitStr>,
+}
+
+impl Parse for MacroArgs {
+    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+        // Parse either one string literal (the asset location) or two comma-separated string
+        // literals (the asset location and the asset subfolder).
+        Ok(MacroArgs {
+            asset_name: input.parse()?,
+            asset_folder: if input.peek(Token![,]) {
+                let _comma: Token![,] = input.parse()?;
+                Some(input.parse()?)
+            } else {
+                None
+            },
+        })
+    }
+}
+
+#[proc_macro]
+pub fn bundled_asset(input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(input as MacroArgs);
+    let asset_name = args.asset_name.value();
+    let asset_folder_arg = args.asset_folder.map(|s| s.value());
+    let asset_folder = asset_folder_arg.as_deref().unwrap_or(BUNDLED_ASSETS_DIR);
+
+    match construct_bundled_asset(&asset_name, asset_folder) {
+        Ok(ok) => ok.into(),
+        Err(err_str) => format_error(&asset_name, asset_folder, err_str).into(),
+    }
+}
+
+fn construct_bundled_asset(asset_name: &str, asset_dir: &str) -> Result<TokenStream2, String> {
+    if full_asset_path(asset_name, asset_dir).exists() {
+        let full_location = format!("{asset_dir}/{asset_name}");
+        Ok(quote! {
+            ::warpui::assets::asset_cache::AssetSource::Bundled {
+                path: #full_location .into(),
+            }
+        })
+    } else {
+        Err("file not found".into())
+    }
+}
+
+#[proc_macro]
+pub fn remote_asset(input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(input as MacroArgs);
+    let asset_name = args.asset_name.value();
+    let asset_folder_arg = args.asset_folder.map(|s| s.value());
+    let asset_folder = asset_folder_arg.as_deref().unwrap_or(REMOTE_ASSETS_DIR);
+
+    match construct_remote_asset(&asset_name, asset_folder) {
+        Ok(ok) => ok.into(),
+        Err(err_str) => format_error(&asset_name, asset_folder, err_str).into(),
+    }
+}
+
+fn construct_remote_asset(asset_name: &str, asset_dir: &str) -> Result<TokenStream2, String> {
+    let full_path = full_asset_path(asset_name, asset_dir);
+    let contents = std::fs::read(full_path).map_err(|err| err.to_string())?;
+
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(&contents);
+    let hash: [u8; 32] = hasher.finalize().into();
+    let url = warp_util::assets::hashed_asset_url(&warp_util::assets::hashed_asset_path(
+        Path::new(asset_name),
+        &hash,
+    ));
+
+    Ok(quote! {
+        ::asset_cache::url_source(::warp_util::assets::make_absolute_url( #url ))
+    })
+}
+
+#[proc_macro]
+pub fn bundled_or_fetched_asset(input: TokenStream) -> TokenStream {
+    // Proc macros are always compiled on the host, and unfortunately they have no way of getting
+    // information about the target of the crate they're being used in (see:
+    // https://github.com/rust-lang/cargo/issues/10714). To work around this, we return
+    // conditionally compiled references to the appropriate macro.
+    let input_lit = parse_macro_input!(input as LitStr);
+
+    // Attributes cannot be used on most expressions, so we make a short block so the attribute can
+    // be applied in a statement context.
+    quote! {
+        {
+            #[cfg(not(target_family = "wasm"))]
+            let val = ::asset_macro::bundled_asset!( #input_lit, #ASYNC_ASSETS_DIR );
+            #[cfg(target_family = "wasm")]
+            let val = ::asset_macro::remote_asset!( #input_lit, #ASYNC_ASSETS_DIR );
+
+            val
+        }
+    }
+    .into()
+}
+
+fn full_asset_path(asset_name: &str, asset_dir: &str) -> PathBuf {
+    // The working directory when running a proc macro is not guaranteed, so we base relative paths
+    // off the location of the cargo manifest.
+    let crate_root =
+        env::var("CARGO_MANIFEST_DIR").expect("missing basic cargo environment variable");
+
+    PathBuf::from(crate_root)
+        .join(ASSETS_DIR)
+        .join(asset_dir)
+        .join(asset_name)
+}
+
+fn format_error(asset_name: &str, asset_dir: &str, error_string: String) -> TokenStream2 {
+    let full_path = full_asset_path(asset_name, asset_dir);
+    let error_message = format!("Error loading asset at {full_path:?}: {error_string}");
+
+    quote! {
+        compile_error!(#error_message)
+    }
+}

crates/channel_versions/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "channel_versions"
+version = "0.1.0"
+edition = "2021"
+publish.workspace = true
+license.workspace = true
+
+[[bin]]
+name = "apply_overrides"
+required-features = ["cli"]
+
+[[bin]]
+name = "version_compare"
+required-features = ["cli"]
+
+[features]
+default = ["cli"]
+cli = ["dep:clap"]
+
+[dependencies]
+anyhow = "1.0"
+chrono = {version = "0.4.23", features = ["serde"]}
+clap = { workspace = true, features = ["derive"], optional = true }
+lazy_static = "1.4.0"
+memo-map = "0.3.1"
+regex = "1"
+serde = {version = "1.0", features = ["derive", "rc"]}
+serde_json.workspace = true

crates/channel_versions/src/bin/apply_overrides.rs

@@ -0,0 +1,51 @@
+use std::{fs, path::PathBuf};
+
+use anyhow::Result;
+use channel_versions::{overrides, ChannelVersion, ChannelVersions};
+use clap::Parser;
+
+#[derive(Parser, Debug)]
+#[command(about)]
+struct Args {
+    /// Name of the operating system to parse the version for.
+    #[arg(long, value_enum)]
+    target_os: channel_versions::overrides::TargetOS,
+
+    /// The file containing a JSON-serialied [`ChannelVersions`] struct to
+    /// apply overrides for.
+    file: PathBuf,
+}
+
+/// Reads in a JSON-serialized [`ChannelVersions`] from a file, applies any
+/// defined overrides that match a given target OS, and prints out the updated
+/// JSON (omitting changelogs).
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    let contents = fs::read_to_string(&args.file)?;
+
+    // Deserialize the JSON data into the expected format.
+    let versions: ChannelVersions = serde_json::from_str(contents.as_str())?;
+
+    let context = overrides::Context {
+        target_os: Some(args.target_os),
+    };
+
+    let dev_version_info = versions.dev.version_info_for_execution_context(&context);
+    let preview_version_info = versions
+        .preview
+        .version_info_for_execution_context(&context);
+    let stable_version_info = versions.stable.version_info_for_execution_context(&context);
+
+    let transformed_versions = ChannelVersions {
+        dev: ChannelVersion::new(dev_version_info),
+        preview: ChannelVersion::new(preview_version_info),
+        stable: ChannelVersion::new(stable_version_info),
+        changelogs: None,
+    };
+
+    // Print out the transformed version info.
+    println!("{}", serde_json::to_string_pretty(&transformed_versions)?);
+
+    Ok(())
+}

crates/channel_versions/src/bin/version_compare.rs

@@ -0,0 +1,42 @@
+use std::process::exit;
+
+use anyhow::Result;
+use channel_versions::ParsedVersion;
+use clap::Parser;
+
+#[derive(Parser, Debug)]
+#[command(about)]
+struct Args {
+    #[arg(long)]
+    version_to_roll_out: String,
+
+    #[arg(long)]
+    current_version: String,
+}
+
+/// Compares two versions and exits with a non-zero exit code if the version to rollout is older than the current version.
+/// Used within the `channel-versions` repo to ensure that we always specify the `is_rollback` field when rolling back.
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    let version_to_roll_out = args.version_to_roll_out;
+    let current_version = args.current_version;
+
+    let parsed_version_to_roll_out = ParsedVersion::try_from(version_to_roll_out.as_str())?;
+    let parsed_current_version = ParsedVersion::try_from(current_version.as_str())?;
+
+    match parsed_version_to_roll_out.cmp(&parsed_current_version) {
+        std::cmp::Ordering::Less => {
+            println!("Current version ({current_version}) is newer than the version to roll out ({version_to_roll_out})");
+            exit(1);
+        }
+        std::cmp::Ordering::Equal => {
+            println!("Version to rollout ({version_to_roll_out}) is equal to the current version ({current_version})");
+        }
+        std::cmp::Ordering::Greater => {
+            println!("Version to rollout ({version_to_roll_out}) is newer than the current version ({current_version})");
+        }
+    }
+
+    Ok(())
+}

crates/channel_versions/src/channel_versions_tests.rs

@@ -0,0 +1,83 @@
+use chrono::NaiveDate;
+
+use super::*;
+
+#[test]
+fn test_parse_version_string() {
+    let version_string = "v0.2023.05.15.08.04.stable_01";
+    let parsed_version: ParsedVersion = version_string
+        .try_into()
+        .expect("version string is parsable");
+    assert_eq!(parsed_version.major, 0);
+    assert_eq!(
+        parsed_version.date,
+        NaiveDate::from_ymd_opt(2023, 5, 15)
+            .unwrap()
+            .and_hms_opt(8, 4, 0)
+            .unwrap()
+    );
+    assert_eq!(parsed_version.patch, 1);
+}
+
+#[test]
+fn test_major_versions_compare_correctly() {
+    let older_version: ParsedVersion = "v0.2023.05.15.08.04.stable_01"
+        .try_into()
+        .expect("older_version is parsable");
+    let newer_version: ParsedVersion = "v1.2023.05.15.08.04.stable_01"
+        .try_into()
+        .expect("newer_version is parsable");
+    assert!(newer_version > older_version);
+}
+
+#[test]
+fn test_dates_compare_correctly() {
+    let older_version: ParsedVersion = "v0.2023.05.15.08.04.stable_01"
+        .try_into()
+        .expect("older_version is parsable");
+    let newer_version: ParsedVersion = "v0.2023.05.22.08.04.stable_00"
+        .try_into()
+        .expect("newer_version is parsable");
+    assert!(newer_version > older_version);
+}
+
+#[test]
+fn test_patches_compare_correctly() {
+    let older_version: ParsedVersion = "v0.2023.05.15.08.04.stable_00"
+        .try_into()
+        .expect("older_version is parsable");
+    let newer_version: ParsedVersion = "v0.2023.05.15.08.04.stable_01"
+        .try_into()
+        .expect("newer_version is parsable");
+    assert!(newer_version > older_version);
+}
+
+#[test]
+fn test_ignores_unknown_channels() {
+    // We no longer support or parse-out beta and canary versions, but we
+    // need to be able to parse a JSON file that still contains them.
+    let channel_version_string = r#"{
+        "beta": {
+          "version": "v0.2024.01.30.16.52.beta_00"
+        },
+        "canary": {
+          "version": "v0.2022.09.29.08.08.canary_00"
+        },
+        "dev": {
+          "version": "v0.2024.01.30.20.34.dev_00"
+        },
+        "preview": {
+          "version": "v0.2024.01.30.20.34.preview_00"
+        },
+        "stable": {
+          "version": "v0.2024.01.16.16.31.stable_01"
+        }
+      }"#;
+
+    let channel_versions: ChannelVersions = serde_json::from_str(channel_version_string)
+        .expect("Should be able to parse channel versions");
+    assert_eq!(
+        channel_versions.stable.version_info().version,
+        "v0.2024.01.16.16.31.stable_01"
+    );
+}

crates/channel_versions/src/lib.rs

@@ -0,0 +1,243 @@
+pub mod overrides;
+
+use std::collections::HashMap;
+use std::fmt::Write;
+
+use anyhow::{Context, Result};
+use chrono::{DateTime, FixedOffset, NaiveDateTime};
+use lazy_static::lazy_static;
+use memo_map::MemoMap;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+
+use overrides::*;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ChannelVersions {
+    pub dev: ChannelVersion,
+    pub preview: ChannelVersion,
+    pub stable: ChannelVersion,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub changelogs: Option<ChannelChangelogs>,
+}
+
+impl std::fmt::Display for ChannelVersions {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "dev: {:?}; preview: {:?}; stable: {:?}",
+            self.dev, self.preview, self.stable
+        )
+    }
+}
+
+lazy_static! {
+    static ref VERSION_RE: Regex = Regex::new(r"v(\d+)\.(.+)\.(.+)_(\d+)").unwrap();
+
+    // Cached mapping of version strings to semantic versions.
+    static ref PARSED_VERSIONS_CACHE: MemoMap<String, ParsedVersion> = Default::default();
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+pub struct ParsedVersion {
+    major: usize,
+    date: NaiveDateTime,
+    patch: usize,
+}
+
+impl TryFrom<&str> for ParsedVersion {
+    type Error = anyhow::Error;
+
+    fn try_from(value: &str) -> Result<Self> {
+        PARSED_VERSIONS_CACHE
+            .get_or_try_insert(value, || {
+                VERSION_RE
+                    .captures(value)
+                    .and_then(|captures| {
+                        let date_str = captures.get(2)?.as_str();
+                        let date =
+                            NaiveDateTime::parse_from_str(date_str, "%Y.%m.%d.%H.%M").ok()?;
+                        Some(ParsedVersion {
+                            major: captures.get(1)?.as_str().parse().ok()?,
+                            date,
+                            patch: captures.get(4)?.as_str().parse().ok()?,
+                        })
+                    })
+                    .context("Can't parse string into Version")
+            })
+            .cloned()
+    }
+}
+
+impl Ord for ParsedVersion {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        (self.major, self.date, self.patch).cmp(&(other.major, other.date, other.patch))
+    }
+}
+
+impl PartialOrd for ParsedVersion {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct ChannelVersion {
+    #[serde(flatten)]
+    version_info: VersionInfo,
+    /// Any overrides which should be applied for this channel.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    overrides: Vec<VersionOverride>,
+}
+
+impl ChannelVersion {
+    pub fn new(version_info: VersionInfo) -> Self {
+        Self {
+            version_info,
+            overrides: vec![],
+        }
+    }
+
+    /// Returns the version information, with any applicable overrides applied
+    /// based on the current execution environment.
+    pub fn version_info(&self) -> VersionInfo {
+        let context = overrides::Context::from_env();
+        self.version_info
+            .with_overrides_applied(&self.overrides, &context)
+    }
+
+    /// Returns the version information, with any applicable overrides applied
+    /// based on the provided context.
+    pub fn version_info_for_execution_context(&self, context: &overrides::Context) -> VersionInfo {
+        self.version_info
+            .with_overrides_applied(&self.overrides, context)
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct VersionInfo {
+    pub version: String,
+    /// The version to download for new users from the download page. This is not used on the client
+    /// other than in the `apply_overrides` binary used from the `channel-versions` repo.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub version_for_new_users: Option<String>,
+    /// The time by which the client needs to be updated, after which
+    /// the user sees a warning banner.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub update_by: Option<DateTime<FixedOffset>>,
+    /// If specified, this field indicates the oldest version of the client that is still
+    /// supported. Any version before this version is not supported and the user should update.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub soft_cutoff: Option<String>,
+    /// If specified, this field indicates the latest client version that has a prominent update.
+    /// Versions before `prominent_update` should display the prominent update UI.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub last_prominent_update: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub is_rollback: Option<bool>,
+    /// The version to use for CLI downloads, falling back to `version` if not set.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cli_version: Option<String>,
+}
+
+impl VersionInfo {
+    pub fn new(version: String) -> Self {
+        Self {
+            version,
+            update_by: None,
+            soft_cutoff: None,
+            last_prominent_update: None,
+            version_for_new_users: None,
+            is_rollback: None,
+            cli_version: None,
+        }
+    }
+
+    /// Returns the CLI version, falling back to the app version if not set.
+    pub fn cli_version(&self) -> &str {
+        self.cli_version.as_deref().unwrap_or(&self.version)
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ChannelChangelogs {
+    // Maps of changelogs by version
+    pub dev: HashMap<String, Changelog>,
+    pub preview: HashMap<String, Changelog>,
+    pub stable: HashMap<String, Changelog>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct Changelog {
+    pub date: DateTime<FixedOffset>,
+    pub sections: Vec<Section>,
+    #[serde(default = "default_markdown_sections")]
+    pub markdown_sections: Vec<MarkdownSection>,
+    #[serde(default)]
+    pub image_url: Option<String>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub oz_updates: Vec<String>,
+}
+
+// Default value for when the changelog JSON doesn't have the markdown_sections field
+fn default_markdown_sections() -> Vec<MarkdownSection> {
+    vec![
+        MarkdownSection {
+            title: "New features".to_string(),
+            markdown: "".to_string(),
+        },
+        MarkdownSection {
+            title: "Improvements".to_string(),
+            markdown: "".to_string(),
+        },
+        MarkdownSection {
+            title: "Coming soon".to_string(),
+            markdown: "".to_string(),
+        },
+    ]
+}
+
+impl std::fmt::Display for Changelog {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            self.sections
+                .iter()
+                .fold(String::new(), |mut output, item| {
+                    let _ = write!(output, "{item}\n\n");
+                    output
+                })
+        )
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct Section {
+    pub title: String,
+    pub items: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct MarkdownSection {
+    pub title: String,
+    pub markdown: String,
+}
+
+impl std::fmt::Display for Section {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}:\n{}",
+            self.title,
+            self.items.iter().fold(String::new(), |mut output, item| {
+                let _ = writeln!(output, "- {item}");
+                output
+            })
+        )
+    }
+}
+
+#[cfg(test)]
+#[path = "channel_versions_tests.rs"]
+mod tests;

crates/channel_versions/src/overrides.rs

@@ -0,0 +1,137 @@
+//! Functionality relating to overrides of the default per-channel version
+//! info.
+//!
+//! For example, we may want to roll out a hotfix release for Linux, but
+//! not want macOS and Windows users to need to perform another update.
+
+use serde::{Deserialize, Serialize};
+
+use crate::VersionInfo;
+
+/// The set of contextual information that is relevant for applying per-version
+/// overrides.
+pub struct Context {
+    pub target_os: Option<TargetOS>,
+}
+
+impl Context {
+    pub fn from_env() -> Self {
+        Context {
+            target_os: TargetOS::current(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "cli", derive(clap::ValueEnum), clap(rename_all = "lower"))]
+pub enum TargetOS {
+    #[serde(rename = "macos")]
+    MacOS,
+    #[serde(rename = "linux")]
+    Linux,
+    #[serde(rename = "windows")]
+    Windows,
+    #[serde(rename = "web")]
+    Web,
+
+    // Catch-all in case we we can't deserialize an unrecognized enum variant.
+    // We need [value(skip)] here to tell `clap` to ignore this variant.
+    #[serde(untagged)]
+    #[cfg_attr(feature = "cli", value(skip))]
+    Unknown(String),
+}
+
+impl TargetOS {
+    /// Returns the current operating system, based on the build-time target_os
+    /// cfg variable, or None if it is not supported.
+    pub fn current() -> Option<Self> {
+        if cfg!(target_family = "wasm") {
+            Some(TargetOS::Web)
+        } else if cfg!(target_os = "macos") {
+            Some(TargetOS::MacOS)
+        } else if cfg!(target_os = "linux") {
+            Some(TargetOS::Linux)
+        } else if cfg!(target_os = "windows") {
+            Some(TargetOS::Windows)
+        } else {
+            None
+        }
+    }
+
+    /// Returns the name of the [`TargetOS`], or None if it is unknown.
+    pub fn name(&self) -> Option<String> {
+        let name = match self {
+            TargetOS::MacOS => "MacOS".to_owned(),
+            TargetOS::Linux => "Linux".to_owned(),
+            TargetOS::Windows => "Windows".to_owned(),
+            TargetOS::Web => "Web".to_owned(),
+            _ => return None,
+        };
+        Some(name)
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+enum OverridePredicate {
+    #[serde(rename = "target_os")]
+    TargetOS(TargetOS),
+}
+
+impl OverridePredicate {
+    fn matches(&self, context: &Context) -> bool {
+        match self {
+            OverridePredicate::TargetOS(os) => {
+                if let Some(target_os) = &context.target_os {
+                    os == target_os
+                } else {
+                    false
+                }
+            }
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct VersionOverride {
+    /// The predicate which determines whether or not this override should be
+    /// applied.
+    predicate: OverridePredicate,
+    /// The overridden version info.
+    version_info: VersionInfo,
+}
+
+impl VersionInfo {
+    /// Returns a copy of this [`VersionInfo`] with the first matching override
+    /// applied, if any match.
+    pub fn with_overrides_applied(&self, overrides: &[VersionOverride], context: &Context) -> Self {
+        let mut new = self.clone();
+        for version_override in overrides {
+            if version_override.predicate.matches(context) {
+                new.apply_override(version_override.version_info.clone());
+                // We only apply the first matching override, and skip the rest.
+                break;
+            }
+        }
+        new
+    }
+
+    fn apply_override(&mut self, other: VersionInfo) {
+        self.version = other.version;
+        if let Some(soft_cutoff) = other.soft_cutoff {
+            self.soft_cutoff = Some(soft_cutoff);
+        }
+        if let Some(update_by) = other.update_by {
+            self.update_by = Some(update_by);
+        }
+        if let Some(last_prominent_update) = other.last_prominent_update {
+            self.last_prominent_update = Some(last_prominent_update);
+        }
+        if let Some(cli_version) = other.cli_version {
+            self.cli_version = Some(cli_version);
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "overrides_tests.rs"]
+mod tests;

crates/channel_versions/src/overrides_tests.rs

@@ -0,0 +1,223 @@
+use chrono::{DateTime, Utc};
+
+use crate::{ChannelVersion, ChannelVersions};
+
+use super::*;
+
+#[test]
+fn test_only_first_override_is_applied() {
+    #[cfg(target_os = "macos")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::MacOS);
+    #[cfg(target_os = "linux")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Linux);
+    #[cfg(target_os = "windows")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Windows);
+
+    let version = ChannelVersion {
+        version_info: VersionInfo {
+            version: "base_version".to_string(),
+            update_by: None,
+            soft_cutoff: None,
+            last_prominent_update: None,
+            is_rollback: None,
+            version_for_new_users: None,
+            cli_version: None,
+        },
+        overrides: vec![
+            VersionOverride {
+                predicate: predicate.clone(),
+                version_info: VersionInfo {
+                    version: "override_version".to_string(),
+                    update_by: Some(DateTime::<Utc>::MIN_UTC.fixed_offset()),
+                    soft_cutoff: Some("override_cutoff".to_string()),
+                    last_prominent_update: None,
+                    is_rollback: None,
+                    version_for_new_users: None,
+                    cli_version: None,
+                },
+            },
+            VersionOverride {
+                predicate,
+                version_info: VersionInfo {
+                    // This should not be applied; as we only apply the first
+                    // matching override.
+                    version: "second_override_version".to_string(),
+                    update_by: None,
+                    soft_cutoff: None,
+                    last_prominent_update: None,
+                    is_rollback: None,
+                    version_for_new_users: None,
+                    cli_version: None,
+                },
+            },
+        ],
+    };
+
+    let version_info = version.version_info.clone();
+    let version_info_with_overrides = version.version_info();
+    assert_ne!(version_info.version, version_info_with_overrides.version);
+    assert_eq!(version_info_with_overrides.version, "override_version");
+    assert_ne!(
+        version_info.update_by,
+        version_info_with_overrides.update_by
+    );
+    assert_eq!(
+        version_info_with_overrides.update_by,
+        Some(DateTime::<Utc>::MIN_UTC.fixed_offset())
+    );
+    assert_ne!(
+        version_info.soft_cutoff,
+        version_info_with_overrides.soft_cutoff
+    );
+    assert_eq!(
+        version_info_with_overrides.soft_cutoff,
+        Some("override_cutoff".to_string())
+    );
+}
+
+#[test]
+fn test_unknown_target_is_ignored() {
+    let channel_version_string = r#"{
+        "beta": {
+          "version": "v0.2024.01.30.16.52.beta_00"
+        },
+        "canary": {
+          "version": "v0.2022.09.29.08.08.canary_00"
+        },
+        "dev": {
+          "soft_cutoff": "v0.2023.05.12.08.03.dev_00",
+          "version": "v0.2024.01.30.20.34.dev_00"
+        },
+        "preview": {
+          "version": "v0.2024.01.30.20.34.preview_00"
+        },
+        "stable": {
+          "soft_cutoff": "v0.2023.11.28.08.02.stable_00",
+          "version": "v0.2024.01.16.16.31.stable_01",
+          "overrides": [
+            {
+              "predicate": {
+                "target_os": "gibberish"
+              },
+              "version_info": {
+                "version": "v0.2024.01.30.16.52.stable_00"
+              }
+            }
+          ]
+        }
+      }"#;
+
+    // We should still be able to deserialize even if the target OS isn't recognized.
+    let channel_versions: ChannelVersions = serde_json::from_str(channel_version_string)
+        .expect("Should be able to parse channel versions");
+    assert_eq!(
+        channel_versions.stable.version_info().version,
+        "v0.2024.01.16.16.31.stable_01"
+    );
+
+    // The override should have no effect, as the target OS is gibberish.
+    let version_with_overrides = channel_versions
+        .stable
+        .version_info()
+        .with_overrides_applied(&channel_versions.stable.overrides, &Context::from_env());
+    assert_eq!(
+        version_with_overrides.version,
+        "v0.2024.01.16.16.31.stable_01"
+    );
+}
+
+#[test]
+fn test_cli_version_override_is_applied() {
+    #[cfg(target_os = "macos")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::MacOS);
+    #[cfg(target_os = "linux")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Linux);
+    #[cfg(target_os = "windows")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Windows);
+
+    let version = ChannelVersion {
+        version_info: VersionInfo {
+            version: "base_version".to_string(),
+            update_by: None,
+            soft_cutoff: None,
+            last_prominent_update: None,
+            is_rollback: None,
+            version_for_new_users: None,
+            cli_version: Some("base_cli_version".to_string()),
+        },
+        overrides: vec![VersionOverride {
+            predicate,
+            version_info: VersionInfo {
+                version: "override_version".to_string(),
+                update_by: None,
+                soft_cutoff: None,
+                last_prominent_update: None,
+                is_rollback: None,
+                version_for_new_users: None,
+                cli_version: Some("override_cli_version".to_string()),
+            },
+        }],
+    };
+
+    let version_info_with_overrides = version.version_info();
+    assert_eq!(
+        version_info_with_overrides.cli_version,
+        Some("override_cli_version".to_string())
+    );
+    assert_eq!(
+        version_info_with_overrides.cli_version(),
+        "override_cli_version"
+    );
+}
+
+#[test]
+fn test_cli_version_preserved_when_override_omits_it() {
+    #[cfg(target_os = "macos")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::MacOS);
+    #[cfg(target_os = "linux")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Linux);
+    #[cfg(target_os = "windows")]
+    let predicate = OverridePredicate::TargetOS(TargetOS::Windows);
+
+    let version = ChannelVersion {
+        version_info: VersionInfo {
+            version: "base_version".to_string(),
+            update_by: None,
+            soft_cutoff: None,
+            last_prominent_update: None,
+            is_rollback: None,
+            version_for_new_users: None,
+            cli_version: Some("base_cli_version".to_string()),
+        },
+        overrides: vec![VersionOverride {
+            predicate,
+            version_info: VersionInfo {
+                version: "override_version".to_string(),
+                update_by: None,
+                soft_cutoff: None,
+                last_prominent_update: None,
+                is_rollback: None,
+                version_for_new_users: None,
+                cli_version: None,
+            },
+        }],
+    };
+
+    let version_info_with_overrides = version.version_info();
+    // cli_version should be preserved from the base since the override doesn't set it.
+    assert_eq!(
+        version_info_with_overrides.cli_version,
+        Some("base_cli_version".to_string())
+    );
+    assert_eq!(
+        version_info_with_overrides.cli_version(),
+        "base_cli_version"
+    );
+}
+
+#[test]
+fn test_cli_version_falls_back_to_version() {
+    let info = VersionInfo::new("app_version".to_string());
+    assert_eq!(info.cli_version, None);
+    assert_eq!(info.cli_version(), "app_version");
+}

crates/command-signatures-v2/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "command-signatures-v2"
+version = "0.1.0"
+edition = "2021"
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+rust-embed.workspace = true
+
+[build-dependencies]
+anyhow.workspace = true
+command.workspace = true

crates/command-signatures-v2/build.rs

@@ -0,0 +1,51 @@
+use command::blocking::Command;
+use std::path::Path;
+
+use anyhow::anyhow;
+
+fn main() -> anyhow::Result<()> {
+    println!("cargo:rerun-if-changed=js/src");
+    println!("cargo:rerun-if-changed=js/build");
+    println!("cargo:rerun-if-changed=js/package.json");
+    println!("cargo:rerun-if-changed=js/tsconfig.json");
+
+    if let Err(e) = build_command_signatures() {
+        if !Path::new(format!("{}/js/build", env!("CARGO_MANIFEST_DIR")).as_str()).exists() {
+            panic!(
+                r#"Failed to build command signatures JS: {e:?}.
+
+Most likely, this is fixed by:
+    1) Ensuring you have an up-to-date Node version; 18.14.1 (required for warp-server development) should suffice.
+    2) Running `corepack enable` (this can be done in any directory).
+    3) Removing a conflicting yarn installed by brew by running:
+        brew uninstall yarn
+
+If you continue to encounter issues, ensure you don't have conflicting Node installations, one of which might not be a sufficiently recent version.
+"#
+            )
+        } else {
+            println!("cargo:warning=Failed to build command signatures JS. Proceeding with stale command signatures!");
+        }
+    }
+    Ok(())
+}
+
+fn build_command_signatures() -> anyhow::Result<()> {
+    match Command::new("yarn")
+        .arg("build")
+        .current_dir(format!("{}/js", env!("CARGO_MANIFEST_DIR")))
+        .output()
+    {
+        Ok(output) => {
+            if output.status.success() {
+                Ok(())
+            } else {
+                Err(anyhow!(
+                    "Failed to build Command Signatures JS with output: {:?}",
+                    output
+                ))
+            }
+        }
+        Err(e) => Err(anyhow::Error::from(e)),
+    }
+}

crates/command-signatures-v2/js/.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{js,json,yml}]
+charset = utf-8
+indent_style = space
+indent_size = 2

crates/command-signatures-v2/js/.gitattributes

@@ -0,0 +1,4 @@
+/.yarn/**            linguist-vendored
+/.yarn/releases/*    binary
+/.yarn/plugins/**/*  binary
+/.pnp.*              binary linguist-generated

crates/command-signatures-v2/js/.gitignore

@@ -0,0 +1,13 @@
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+# Swap the comments on the following lines if you wish to use zero-installs
+# In that case, don't forget to run `yarn config set enableGlobalCache false`!
+# Documentation here: https://yarnpkg.com/features/zero-installs
+
+!.yarn/cache
+# .pnp.*

crates/command-signatures-v2/js/.yarnrc.yml

@@ -0,0 +1,3 @@
+enableGlobalCache: false
+
+nodeLinker: pnp

crates/command-signatures-v2/js/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "description": "Warp Command Signatures",
+  "main": "build/main.js",
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "watch": "npm run build -- --watch",
+    "clean": "rm -r build"
+  },
+  "author": "eng@warp.dev",
+  "license": "",
+  "devDependencies": {
+    "typescript": "^5.2.2"
+  },
+  "packageManager": "yarn@4.0.1"
+}

crates/command-signatures-v2/js/src/main.ts

@@ -0,0 +1,71 @@
+let jackLastNameArgument: Argument = {
+  name: "last-name",
+  description: "Last name of the jack",
+  values: [{ value: "Nichols" }, { value: "Nicholson" }],
+};
+let generatedArgument: Argument = {
+    name: "generated-name",
+    description: "Last name of the jack",
+    values: [
+      {
+        generateSuggestionsFn: { script: "echo foobar" }
+      }
+    ],
+};
+
+export function activate(warp: Warp) {
+  warp.completions.registerCommandSignature({
+    command: {
+      name: "jack",
+      alias: "j",
+      description: "Jack's special command",
+      subcommands: [
+        {
+          name: "create",
+          description: "Create a jack :D.",
+          arguments: jackLastNameArgument,
+        },
+        {
+          name: "delete",
+          description: "Delete a jack :(.",
+          arguments: jackLastNameArgument,
+        },
+        {
+          name: "update",
+          description: "Update a jack",
+          arguments: [
+            jackLastNameArgument,
+          ]
+        },
+        {
+          name: "generate",
+          description: "Generate a jack",
+          arguments: [
+            generatedArgument
+          ]
+        },
+      ],
+      options: [
+        {
+          name: ["-s", "--special"],
+          description: "Only operate on special jacks",
+          arguments: {
+            name: "level",
+            description: "Level of specialness",
+            values: [ { value: "1" }, { value: "2" }, { value: "3" }],
+          }
+        },
+        {
+          name: ["-c", "--cracker"],
+          description: "Only operate on cracker jacks",
+        },
+        {
+          name: "--eat",
+          description: "Eat the jacks",
+          requiredOptions: ["-c"],
+        },
+      ],
+    },
+  });
+  console.log("Successfully registered command signatures!");
+}

crates/command-signatures-v2/js/src/types/command-signature.d.ts

@@ -0,0 +1,161 @@
+// A union type to express common command token delimiters. This is used by various 
+// fields throughout the schema.
+type Delimiter = ',' | ':' | ';' | '/' | ' ' | '::';
+
+// Add a new wrapper struct around the root Command object, which scales to provide a 
+// semantically correct place for whole-command level configuration properties, like
+// supported CLI versions, signature author(s), way to resolve conflicts among CLIs with 
+// the same name. 
+interface CommandSignature {
+  command: Command;
+
+// See More configurable options parsing below.
+  parseOptions?: {
+    optionArgumentDelimiters?: Delimiter[];
+    optionsMustPrecedeArguments?: boolean;
+    flagsArePosixNoncompliant?: boolean;
+  };
+}
+
+// This is passed to generateAdditionalSuggestions and custom generator implementations.
+// See Support dynamic commands and Custom generators below.
+interface CompletionContext {  
+  // Tokens in the input buffer.
+  tokens: string[];
+
+  // Path to shell executable.
+  shell: string;
+
+  // The session's current working directory.
+  pwd: string;
+  
+  // Way to execute arbitrary shell command.
+  // Maybe restrict this to running in restricted mode? How do we ensure this is safe?
+  executeShellCommand: (command: string) => Promise<{
+    exitCode: number;
+    success: boolean;
+    output: string;
+  }>;
+}
+
+interface Command {
+  name: string;
+  alias?: string | string[];
+
+  // See Configure suggestion accept behavior below.
+  insertValue?: string;
+
+  description?: string;
+  arguments?: Argument | Argument[];
+  subcommands?: Command[];
+  options?: Option[];
+  priority?: number;
+
+  // See Support dynamic subcommands below.
+  runtimeOptionsAndSubcommands?: (ctx: CompletionContext) => {
+  	options?: Option[];
+    subcommands?: Command[];
+  };
+}
+
+interface Argument {
+  name: string;
+  description?: string;
+  // This is renamed from ArgumentType to ArgumentValue, because that's semantically what 
+  // this array is -- a collection of suggestions for argument values.
+  values?: ArgumentValue[];
+  optional?: boolean;
+
+  // See Alias support below.
+  expandAlias?: (ctx: CompletionContext) => Promise<string[]>;
+
+  // See Multi-suggestion arguments below.
+  arity?: {
+    limit?: number | undefined,
+    delimiter?: Delimiter[],
+  }, 
+}
+
+declare enum TemplateType {
+  Files = "TemplateType.Files",
+  Folders = "TemplateType.Folders" ,
+  FilesAndFolders = "TemplateType.FilesAndFolders",
+}
+
+interface Template {
+  typeName: TemplateType;
+  filterName?: string;
+}
+
+interface ShellCommandGeneratorFn {
+  script: string | ((tokens: string[]) => string);
+
+  // If left unspecified, splits the output of script on newlines by default. Splitting 
+  // on newlines by default is new behavior (currently the postProcess function is 
+  // required).
+  postProcess?: (script_output: string) => GeneratorResults;
+}
+
+// See Custom generators below.
+type CustomGeneratorFn = (ctx: CompletionContext) => Promise<GeneratorResults>;
+type GeneratorFn = ShellCommandGeneratorFn | CustomGeneratorFn;
+
+interface SuggestionGenerator {
+  generateSuggestionsFn: GeneratorFn,
+
+  options?: {
+    customTrigger?: string
+  }
+}
+
+interface GeneratorResults {
+  suggestions: Suggestion[];
+  is_ordered?: boolean;
+}
+
+declare type RootCommand = {
+  is_root_command: true
+};
+
+type ArgumentValue = Suggestion
+  | Template
+  | SuggestionGenerator
+  | RootCommand;
+
+interface Suggestion {
+  value: string;
+  displayValue?: string;
+  description?: string;
+  priority?: number;
+  icon?: IconType;
+  isHidden?: boolean;
+
+  insertValue?: string;
+
+  dangerous?: boolean;
+  deprecated?: boolean;
+}
+
+declare enum IconType {
+  File = "IconType.File",
+  Folder = "IconType.Folder",
+  GitBranch = "IconType.GitBranch",
+}
+
+interface Option {
+  name: string | string[];
+
+  insertValue?: string;
+  description?: string;
+  arguments?: Argument | Argument[];
+  required?: boolean;
+  priority?: number;
+
+  dangerous?: boolean;
+  deprecated?: boolean;
+
+  requiredOptions?: string[];
+  incompatibleOptions?: string[];
+  repeatable?: boolean;
+}
+

crates/command-signatures-v2/js/src/types/warp.d.ts

@@ -0,0 +1,12 @@
+interface Warp {
+  completions: Completions,
+}
+
+interface Completions {
+  registerCommandSignature: (signatures: CommandSignature | CommandSignature[]) => void,
+}
+
+declare namespace console {
+  function log(message: string): void;
+  function err(message: string): void;
+}

crates/command-signatures-v2/js/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "include": ["src/**/*"],
+  "compilerOptions": {
+    "target": "es2019",
+    "lib": ["es2019"],
+    "outDir": "build",
+    "strict": true,
+    "typeRoots": [
+      "./src/types"
+    ]
+  }
+}
+

crates/command-signatures-v2/js/yarn.lock

@@ -0,0 +1,34 @@
+# This file is generated by running "yarn install" inside your project.
+# Manual changes might be lost - proceed with caution!
+
+__metadata:
+  version: 8
+  cacheKey: 10c0
+
+"js@workspace:.":
+  version: 0.0.0-use.local
+  resolution: "js@workspace:."
+  dependencies:
+    typescript: "npm:^5.2.2"
+  languageName: unknown
+  linkType: soft
+
+"typescript@npm:^5.2.2":
+  version: 5.2.2
+  resolution: "typescript@npm:5.2.2"
+  bin:
+    tsc: bin/tsc
+    tsserver: bin/tsserver
+  checksum: 91ae3e6193d0ddb8656d4c418a033f0f75dec5e077ebbc2bd6d76439b93f35683936ee1bdc0e9cf94ec76863aa49f27159b5788219b50e1cd0cd6d110aa34b07
+  languageName: node
+  linkType: hard
+
+"typescript@patch:typescript@npm%3A^5.2.2#optional!builtin<compat/typescript>":
+  version: 5.2.2
+  resolution: "typescript@patch:typescript@npm%3A5.2.2#optional!builtin<compat/typescript>::version=5.2.2&hash=f3b441"
+  bin:
+    tsc: bin/tsc
+    tsserver: bin/tsserver
+  checksum: 062c1cee1990e6b9419ce8a55162b8dc917eb87f807e4de0327dbc1c2fa4e5f61bc0dd4e034d38ff541d1ed0479b53bcee8e4de3a4075c51a1724eb6216cb6f5
+  languageName: node
+  linkType: hard

crates/command-signatures-v2/src/lib.rs

@@ -0,0 +1,7 @@
+use rust_embed::RustEmbed;
+
+#[derive(Clone, Copy, RustEmbed)]
+#[folder = "js/build"]
+pub struct CommandSignaturesJs;
+
+pub static COMMAND_SIGNATURES_JS: CommandSignaturesJs = CommandSignaturesJs;

crates/command/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "command"
+version = "0.1.0"
+edition = "2024"
+publish.workspace = true
+license.workspace = true
+
+[features]
+test-util = []
+
+[dependencies]
+
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
+async-process = { workspace = true }
+futures-lite.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+anyhow.workspace = true
+lazy_static.workspace = true
+log.workspace = true
+thiserror.workspace = true
+win32job = "2.0.2"
+windows.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+libc.workspace = true