public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce indexing overhead and repair agent search and child recovery

Browse Source
This commit is contained in:
rkw6086
2026-09-14 16:25:49 -05:00
parent 81b4c78b61
commit 506dc5b225
38 changed files with 1941 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ai/src/agent/action_result/convert.rs
+5
View File
@@ -306,6 +306,11 @@ impl TryFrom<SearchCodebaseResult> for api::request::input::tool_call_result::Re
files: files
.into_iter()
.flat_map(Into::<Vec<api::FileContent>>::into)
.map(|mut file| {
let range = file.line_range.as_ref().map(|range| format!(" (lines {}-{})", range.start, range.end)).unwrap_or_default();
file.content = format!("Bounded search excerpt{range}; more matches/content may be omitted. Use read_files for focused follow-up.\n{}", file.content);
file
})
.collect(),
},
)),
crates/ai/src/agent/action_result/mod.rs
+3
View File
@@ -878,6 +878,9 @@ pub enum SearchCodebaseFailureReason {
MissingCurrentWorkingDirectory,
}
/// Maximum number of files returned by a codebase discovery search.
pub const MAX_SEARCH_CODEBASE_FILES: usize = 8;
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum SearchCodebaseResult {
Success {
crates/ai/src/index/file_outline/mod.rs
+1
View File
@@ -4,6 +4,7 @@ cfg_if::cfg_if! {
if #[cfg(not(target_arch = "wasm32"))] {
mod native;
pub use native::build_outline;
pub(crate) use native::parse_file_outline_content;
}
}
crates/ai/src/index/file_outline/native.rs
+11 -4
View File
@@ -232,19 +232,26 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
if !is_file_parsable(path)? {
return Err(anyhow!("File exceeds max file size limit for parsing"));
}
let content = fs::read_to_string(path)?;
parse_file_outline_content(path, &content)
}
pub(crate) fn parse_file_outline_content(
path: &Path,
content: &str,
) -> anyhow::Result<FileOutline> {
let standardized_path = StandardizedPath::try_from_local(path)?;
let Some(language) = languages::language_by_filename(&standardized_path) else {
return Err(anyhow!("Language unsupported for file {:?}", path));
};
let content = fs::read_to_string(path)?;
let mut parser = Parser::new();
parser.set_language(&language.grammar)?;
let Some(tree) = parser.parse(&content, None) else {
let Some(tree) = parser.parse(content, None) else {
return Err(anyhow!("Couldn't parse AST"));
};
let symbols = language.symbols_query.as_ref().map(|query| {
get_symbols(query, &tree, &content)
get_symbols(query, &tree, content)
.into_iter()
.map(|(fn_name, type_prefix, comments, line_number)| Symbol {
name: fn_name.to_owned(),
@@ -285,7 +292,7 @@ fn parse_file_outline(path: &Path) -> anyhow::Result<FileOutline> {
fn get_symbols<'a>(
query: &'a Query,
tree: &Tree,
file_content: &'a String,
file_content: &'a str,
) -> Vec<(&'a str, Option<&'a str>, Vec<&'a str>, usize)> {
struct PendingComment<'a> {
lines: Vec<&'a str>,
crates/ai/src/index/local_project_index/mod.rs
+359 -88
View File
@@ -7,6 +7,7 @@
use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::time::Duration;
@@ -16,13 +17,15 @@ use galaxy_util::standardized_path::StandardizedPath;
use galaxyui_core::r#async::Timer;
use galaxyui_core::{Entity, ModelContext, SingletonEntity};
use itertools::Itertools;
use repo_metadata::{RepoMetadataEvent, RepositoryIdentifier};
use repo_metadata::entry::{BudgetExceededBehavior, IgnoredPathStrategy};
use repo_metadata::file_tree_update::RepoNodeMetadata;
use repo_metadata::{Entry, MetadataUpdateType, RepoMetadataEvent, RepositoryIdentifier};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use warp_search_core::define_search_schema;
use warp_search_core::searcher::{SimpleFullTextSearcher, DEFAULT_MEMORY_BUDGET};
use crate::index::build_outline;
use crate::index::file_outline::parse_file_outline_content;
const INDEX_SCHEMA_VERSION: u32 = 1;
const MAX_INDEX_FILES: usize = 5_000;
@@ -31,6 +34,7 @@ const MAX_INDEXED_BODY_BYTES: usize = 256_000;
const INDEX_DIRECTORY_NAME: &str = "local_project_indices";
const CURRENT_FILE_NAME: &str = "CURRENT";
const METADATA_FILE_NAME: &str = "metadata.json";
const UPDATE_DEBOUNCE: Duration = Duration::from_millis(300);
const INDEX_LOCK_RETRY_INTERVAL: Duration = Duration::from_millis(50);
// Field weights intentionally prioritize symbols and paths over implementation text.
@@ -108,10 +112,18 @@ impl LocalProjectIndex {
}
}
/// Paths are re-read at execution time, so repeated saves and rename sequences coalesce.
#[derive(Default)]
struct PendingUpdate {
full_rebuild: bool,
paths: HashSet<PathBuf>,
}
pub struct LocalProjectIndexManager {
indices: HashMap<PathBuf, LocalProjectIndex>,
statuses: HashMap<PathBuf, LocalIndexStatus>,
pending_rebuilds: HashSet<PathBuf>,
pending_updates: HashMap<PathBuf, PendingUpdate>,
scheduled_updates: HashSet<PathBuf>,
rebuild_epochs: HashMap<PathBuf, u64>,
storage_root: PathBuf,
}
@@ -125,15 +137,13 @@ impl SingletonEntity for LocalProjectIndexManager {}
impl LocalProjectIndexManager {
pub fn new(ctx: &mut ModelContext<Self>) -> Self {
let repo_metadata = repo_metadata::RepoMetadataModel::handle(ctx);
let manager = Self::new_at(state_dir().join(INDEX_DIRECTORY_NAME), ctx);
let mut manager = Self::new_at(state_dir().join(INDEX_DIRECTORY_NAME), ctx);
ctx.subscribe_to_model(&repo_metadata, |manager, _, event, ctx| {
manager.handle_repo_metadata_event(event, ctx);
});
// Re-register restored roots with repository metadata so filesystem updates continue to
// refresh the local index after an app restart. RepositoryUpdated is intentionally ignored
// below; the restored generation is already usable and only file-tree updates need a
// rebuild.
// Re-register watchers and reconcile changes made while the app was closed. The restored
// snapshot remains searchable during this one-time rebuild.
let restored_roots: Vec<PathBuf> = manager.statuses.keys().cloned().collect();
for root_path in restored_roots {
let Ok(standardized_root) = StandardizedPath::from_local_canonicalized(&root_path)
@@ -143,6 +153,7 @@ impl LocalProjectIndexManager {
let _ = repo_metadata.update(ctx, |model, ctx| {
model.index_local_directory_path(&standardized_root, ctx)
});
manager.start_rebuild(root_path, ctx);
}
manager
}
@@ -152,7 +163,8 @@ impl LocalProjectIndexManager {
let mut manager = Self {
indices: HashMap::new(),
statuses: HashMap::new(),
pending_rebuilds: HashSet::new(),
pending_updates: HashMap::new(),
scheduled_updates: HashSet::new(),
rebuild_epochs: HashMap::new(),
storage_root,
};
@@ -179,7 +191,10 @@ impl LocalProjectIndexManager {
self.statuses.get(&root_path),
Some(LocalIndexStatus::Indexing)
) {
self.pending_rebuilds.insert(root_path);
self.pending_updates
.entry(root_path)
.or_default()
.full_rebuild = true;
return Ok(());
}
@@ -196,6 +211,47 @@ impl LocalProjectIndexManager {
}
fn start_rebuild(&mut self, root_path: PathBuf, ctx: &mut ModelContext<Self>) {
self.start_update(
root_path,
PendingUpdate {
full_rebuild: true,
paths: HashSet::new(),
},
ctx,
);
}
fn schedule_update(&mut self, root_path: PathBuf, ctx: &mut ModelContext<Self>) {
if matches!(
self.statuses.get(&root_path),
Some(LocalIndexStatus::Indexing)
) || !self.pending_updates.contains_key(&root_path)
|| !self.scheduled_updates.insert(root_path.clone())
{
return;
}
let epoch = self.rebuild_epochs.get(&root_path).copied();
ctx.spawn(
async { Timer::after(UPDATE_DEBOUNCE).await },
move |manager, _, ctx| {
if manager.rebuild_epochs.get(&root_path).copied() != epoch {
return;
}
manager.scheduled_updates.remove(&root_path);
if let Some(update) = manager.pending_updates.remove(&root_path) {
manager.start_update(root_path, update, ctx);
}
},
);
}
fn start_update(
&mut self,
root_path: PathBuf,
update: PendingUpdate,
ctx: &mut ModelContext<Self>,
) {
self.scheduled_updates.remove(&root_path);
let rebuild_epoch = {
let epoch = self.rebuild_epochs.entry(root_path.clone()).or_default();
*epoch += 1;
@@ -210,12 +266,34 @@ impl LocalProjectIndexManager {
let storage_root = self.storage_root.clone();
ctx.spawn(
async move { build_persisted_index(root_path, storage_root).await },
async move {
if update.full_rebuild {
build_persisted_index(root_path, storage_root).await
} else {
match update_persisted_index(
root_path.clone(),
storage_root.clone(),
update.paths,
)
.await
{
Ok(result) => Ok(result),
Err(error) => {
log::warn!(
"Incremental local index update failed; rebuilding: {error:#}"
);
build_persisted_index(root_path, storage_root).await
}
}
}
},
move |manager, result, ctx| match result {
Ok((root_path, built_index, file_count)) => {
if manager.rebuild_epochs.get(&root_path).copied() != Some(rebuild_epoch) {
drop(built_index.searcher);
let _ = fs::remove_dir_all(&built_index.generation_directory);
if built_index.new_generation {
let _ = fs::remove_dir_all(&built_index.generation_directory);
}
return;
}
if let Err(error) = publish_generation(
@@ -241,12 +319,10 @@ impl LocalProjectIndexManager {
root_path: root_path.clone(),
status,
});
let should_rebuild = manager.pending_rebuilds.remove(&root_path);
cleanup_old_generations(&manager.storage_root, &root_path);
drop(built_index.repository_lock);
if should_rebuild {
manager.start_rebuild(root_path, ctx);
}
manager.schedule_update(root_path, ctx);
}
Err((root_path, error)) => {
if manager.rebuild_epochs.get(&root_path).copied() != Some(rebuild_epoch) {
@@ -289,10 +365,7 @@ impl LocalProjectIndexManager {
status,
});
}
let should_rebuild = self.pending_rebuilds.remove(&root_path);
if should_rebuild {
self.start_rebuild(root_path, ctx);
}
self.schedule_update(root_path, ctx);
}
/// Returns the most specific indexed root containing `path`.
@@ -362,15 +435,65 @@ impl LocalProjectIndexManager {
event: &RepoMetadataEvent,
ctx: &mut ModelContext<Self>,
) {
let local_root = match event {
match event {
RepoMetadataEvent::FileTreeEntryUpdated {
id: RepositoryIdentifier::Local(path),
..
} => path.to_local_path(),
update_type,
} => {
let Some(root) = path.to_local_path() else {
return;
};
let root = dunce::canonicalize(&root).unwrap_or(root);
if !self.statuses.contains_key(&root) {
return;
}
let pending = self.pending_updates.entry(root.clone()).or_default();
match update_type {
MetadataUpdateType::FullReplace => pending.full_rebuild = true,
MetadataUpdateType::IncrementalUpdate(update) if !pending.full_rebuild => {
pending.paths.extend(
update
.remove_entries
.iter()
.filter_map(StandardizedPath::to_local_path),
);
for entry in &update.update_entries {
for node in &entry.subtree_metadata {
let path = match node {
RepoNodeMetadata::File(file) => &file.path,
RepoNodeMetadata::Directory(directory) => &directory.path,
};
if let Some(path) = path.to_local_path() {
pending.paths.insert(path);
}
}
}
pending.paths.retain(|path| {
path.starts_with(&root)
&& !path.components().any(|part| part.as_os_str() == ".git")
});
if pending.paths.len() > MAX_INDEX_FILES
|| pending.paths.iter().any(|path| {
path.file_name().is_some_and(|name| name == ".gitignore")
})
{
pending.full_rebuild = true;
}
}
MetadataUpdateType::IncrementalUpdate(_) => {}
}
if pending.full_rebuild {
pending.paths.clear();
}
if !pending.full_rebuild && pending.paths.is_empty() {
self.pending_updates.remove(&root);
return;
}
self.schedule_update(root, ctx);
}
// FileTreeUpdated is emitted before repository mutations are applied. The
// post-application FileTreeEntryUpdated event below is the rebuild trigger; using
// both would start duplicate full rebuilds for every watcher batch.
RepoMetadataEvent::FileTreeUpdated { .. } => None,
// post-application delta is the update trigger; using both duplicates each batch.
RepoMetadataEvent::FileTreeUpdated { .. } => (),
RepoMetadataEvent::RepositoryRemoved {
id: RepositoryIdentifier::Local(path),
} => {
@@ -380,7 +503,6 @@ impl LocalProjectIndexManager {
self.remove_index(&root_path, ctx);
ctx.emit(LocalProjectIndexEvent::IndexRemoved { root_path });
}
return;
}
RepoMetadataEvent::RepositoryUpdated { .. }
| RepoMetadataEvent::RepositoryRemoved {
@@ -392,22 +514,7 @@ impl LocalProjectIndexManager {
}
| RepoMetadataEvent::UpdatingRepositoryFailed { .. }
| RepoMetadataEvent::StandingQueryResultsUpdated { .. }
| RepoMetadataEvent::IncrementalUpdateReady { .. } => None,
};
let Some(local_root) = local_root else {
return;
};
let canonical_root = dunce::canonicalize(&local_root).unwrap_or(local_root);
if self.statuses.contains_key(&canonical_root) {
if matches!(
self.statuses.get(&canonical_root),
Some(LocalIndexStatus::Indexing)
) {
self.pending_rebuilds.insert(canonical_root);
} else {
self.start_rebuild(canonical_root, ctx);
}
| RepoMetadataEvent::IncrementalUpdateReady { .. } => (),
}
}
@@ -415,7 +522,8 @@ impl LocalProjectIndexManager {
let root_path = dunce::canonicalize(root_path).unwrap_or_else(|_| root_path.to_path_buf());
self.indices.remove(&root_path);
self.statuses.remove(&root_path);
self.pending_rebuilds.remove(&root_path);
self.pending_updates.remove(&root_path);
self.scheduled_updates.remove(&root_path);
let epoch = self.rebuild_epochs.entry(root_path.clone()).or_default();
*epoch += 1;
let storage_root = self.storage_root.clone();
@@ -486,20 +594,19 @@ impl LocalProjectIndexManager {
&generation_path,
) {
Ok(searcher) => {
let Ok(file_count) = searcher.get_all_doc_ids().map(|ids| ids.len()) else {
continue;
};
self.indices.insert(
root_path.clone(),
LocalProjectIndex {
root_path: root_path.clone(),
searcher,
file_count: metadata.file_count,
},
);
self.statuses.insert(
root_path,
LocalIndexStatus::Ready {
file_count: metadata.file_count,
file_count,
},
);
self.statuses
.insert(root_path, LocalIndexStatus::Ready { file_count });
}
Err(error) => {
log::warn!(
@@ -518,6 +625,7 @@ struct BuiltLocalProjectIndex {
generation: String,
generation_directory: PathBuf,
repository_lock: File,
new_generation: bool,
}
async fn build_persisted_index(
@@ -531,6 +639,7 @@ async fn build_persisted_index(
// one process cannot delete another process's in-progress Tantivy generation.
let repository_lock = acquire_repository_lock(&storage_root, &root_path).await?;
let documents = build_documents(&root_path).await?;
let file_count = documents.len();
let index_directory = repository_storage_directory(&storage_root, &root_path);
fs::create_dir_all(index_directory.join("generations"))?;
let generation = format!("generation-{}", uuid::Uuid::new_v4());
@@ -552,7 +661,7 @@ async fn build_persisted_index(
return Err(error);
}
};
if let Err(error) = searcher.build_index(build_search_documents(&documents)) {
if let Err(error) = searcher.build_index(build_search_documents(documents)) {
let _ = fs::remove_dir_all(&temporary_directory);
return Err(error);
}
@@ -571,7 +680,7 @@ async fn build_persisted_index(
root_path: root_path.to_string_lossy().into_owned(),
schema_version: INDEX_SCHEMA_VERSION,
generation: generation.clone(),
file_count: documents.len(),
file_count,
};
let metadata_json = serde_json::to_vec_pretty(&metadata)?;
// Store metadata inside the generation so an interrupted refresh cannot replace the
@@ -585,8 +694,9 @@ async fn build_persisted_index(
generation,
generation_directory,
repository_lock,
new_generation: true,
},
documents.len(),
file_count,
))
}
.await;
@@ -602,11 +712,10 @@ struct LocalProjectDocument {
}
fn build_search_documents(
documents: &[LocalProjectDocument],
) -> impl Iterator<Item = LocalProjectSearchDocument> + '_ {
documents: Vec<LocalProjectDocument>,
) -> impl Iterator<Item = LocalProjectSearchDocument> {
documents
.iter()
.cloned()
.into_iter()
.map(|document| LocalProjectSearchDocument {
symbol: document.symbol,
path: document.file_path.clone(),
@@ -617,42 +726,204 @@ fn build_search_documents(
}
async fn build_documents(root_path: &Path) -> Result<Vec<LocalProjectDocument>> {
let root_path = dunce::canonicalize(root_path).with_context(|| {
format!(
"Failed to canonicalize project root {}",
root_path.display()
)
})?;
let outline = build_outline(&root_path, Some(MAX_INDEX_FILES)).await?;
let root_path = dunce::canonicalize(root_path)?;
documents_under_path(&root_path, &root_path, MAX_INDEX_FILES)
}
/// Enumerate only the affected subtree, preserving the outline's existing ignore policy.
fn documents_under_path(
root: &Path,
path: &Path,
limit: usize,
) -> Result<Vec<LocalProjectDocument>> {
if !path.starts_with(root)
|| limit == 0
|| path.components().any(|part| part.as_os_str() == ".git")
{
return Ok(Vec::new());
}
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(error.into()),
};
if metadata.file_type().is_symlink() {
return Ok(Vec::new());
}
// A watcher path below a symlink must follow the same exclusion policy as a full scan.
for ancestor in path
.ancestors()
.skip(1)
.take_while(|ancestor| ancestor.starts_with(root) && *ancestor != root)
{
if fs::symlink_metadata(ancestor)?.file_type().is_symlink() {
return Ok(Vec::new());
}
}
let mut ignores = repo_metadata::gitignores_for_directory(root);
if let Some(parent) = path.parent() {
let mut ancestors: Vec<_> = parent
.ancestors()
.take_while(|ancestor| ancestor.starts_with(root) && *ancestor != root)
.collect();
ancestors.reverse();
for ancestor in ancestors {
let ignore_path = ancestor.join(".gitignore");
if ignore_path.is_file() {
ignores.push(ignore::gitignore::Gitignore::new(ignore_path).0);
}
}
}
if repo_metadata::matches_gitignores(path, metadata.is_dir(), &ignores, true) {
return Ok(Vec::new());
}
let mut files = Vec::new();
let mut budget = limit;
Entry::build_tree(
path,
&mut files,
&mut ignores,
Some(&mut budget),
200,
0,
&IgnoredPathStrategy::Exclude,
BudgetExceededBehavior::StopAndLazyLoad,
)?;
let mut documents = Vec::new();
for file in outline.to_file_symbols(None) {
let relative_path = PathBuf::from(&file.path);
let absolute_path = root_path.join(&relative_path);
let Ok(bytes) = fs::read(&absolute_path) else {
continue;
};
if bytes.is_empty() || bytes.len() > MAX_INDEXED_FILE_BYTES {
continue;
for file in files {
if let Some(document) = build_document(&file.path.to_local_path_lossy())? {
documents.push(document);
}
let Ok(content) = String::from_utf8(bytes) else {
continue;
};
let body = content.chars().take(MAX_INDEXED_BODY_BYTES).collect();
let extension = absolute_path
.extension()
.and_then(|extension| extension.to_str())
.unwrap_or_default();
let metadata = format!("{extension} {}", file.symbols);
documents.push(LocalProjectDocument {
file_path: absolute_path.to_string_lossy().into_owned(),
symbol: file.symbols,
metadata,
body,
});
}
Ok(documents)
}
fn build_document(path: &Path) -> Result<Option<LocalProjectDocument>> {
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error)
if matches!(
error.kind(),
std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied
) =>
{
return Ok(None)
}
Err(error) => return Err(error.into()),
};
if !metadata.is_file() || metadata.len() > MAX_INDEXED_FILE_BYTES as u64 {
return Ok(None);
}
let file = match File::open(path) {
Ok(file) => file,
Err(error)
if matches!(
error.kind(),
std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied
) =>
{
return Ok(None)
}
Err(error) => return Err(error.into()),
};
// The read limit also covers files growing between stat and read.
let mut bytes = Vec::new();
file.take(MAX_INDEXED_FILE_BYTES as u64 + 1)
.read_to_end(&mut bytes)?;
if bytes.is_empty() || bytes.len() > MAX_INDEXED_FILE_BYTES {
return Ok(None);
}
let Ok(mut content) = String::from_utf8(bytes) else {
return Ok(None);
};
let symbol = parse_file_outline_content(path, &content)
.unwrap_or_default()
.to_string()
.unwrap_or_default();
let mut end = content.len().min(MAX_INDEXED_BODY_BYTES);
while !content.is_char_boundary(end) {
end -= 1;
}
content.truncate(end);
content.shrink_to_fit();
let extension = path
.extension()
.and_then(|extension| extension.to_str())
.unwrap_or_default();
Ok(Some(LocalProjectDocument {
file_path: path.to_string_lossy().into_owned(),
metadata: format!("{extension} {symbol}"),
symbol,
body: content,
}))
}
async fn update_persisted_index(
root_path: PathBuf,
storage_root: PathBuf,
paths: HashSet<PathBuf>,
) -> Result<(PathBuf, BuiltLocalProjectIndex, usize)> {
let repository_lock = acquire_repository_lock(&storage_root, &root_path).await?;
let directory = repository_storage_directory(&storage_root, &root_path);
let metadata = read_metadata(&directory)?;
anyhow::ensure!(
metadata.schema_version == INDEX_SCHEMA_VERSION,
"Local index schema changed"
);
let generation_directory = directory.join("generations").join(&metadata.generation);
let searcher = SimpleFullTextSearcher::open_in_dir(
&LOCAL_PROJECT_INDEX_SCHEMA,
DEFAULT_MEMORY_BUDGET,
&generation_directory,
)?;
let existing = searcher.get_all_doc_ids()?;
// Collapse directory/child events so no subtree is parsed more than once per batch.
let mut paths: Vec<_> = paths
.into_iter()
.filter(|path| path.starts_with(&root_path))
.collect();
paths.sort();
let mut roots: Vec<PathBuf> = Vec::new();
for path in paths {
if !roots.iter().any(|root| path.starts_with(root)) {
roots.push(path);
}
}
let existing_count = existing.len();
let deleted: Vec<_> = existing
.into_iter()
.filter(|id| {
roots
.iter()
.any(|path| Path::new(&id.file_path).starts_with(path))
})
.collect();
let retained = existing_count - deleted.len();
let mut documents = Vec::new();
for path in roots {
documents.extend(documents_under_path(
&root_path,
&path,
MAX_INDEX_FILES.saturating_sub(retained + documents.len()),
)?);
}
if !deleted.is_empty() || !documents.is_empty() {
searcher.update_documents(deleted, build_search_documents(documents))?;
}
let file_count = searcher.get_all_doc_ids()?.len();
Ok((
root_path,
BuiltLocalProjectIndex {
searcher,
generation: metadata.generation,
generation_directory,
repository_lock,
new_generation: false,
},
file_count,
))
}
fn format_storage_directory_name(root_path: &Path) -> String {
let mut hasher = Sha256::new();
hasher.update(root_path.to_string_lossy().as_bytes());
crates/ai/src/index/local_project_index/tests.rs
+127
View File
@@ -86,3 +86,130 @@ fn partial_paths_filter_local_search_results() {
assert_eq!(matches.len(), 1);
assert!(matches[0].path.ends_with("src/lib.rs"));
}
#[test]
fn incremental_commit_replaces_and_removes_files_without_reparsing_unchanged_files() {
let root = TempDir::new().unwrap();
let storage = TempDir::new().unwrap();
let root = dunce::canonicalize(root.path()).unwrap();
fs::write(root.join("changed.rs"), "fn old_symbol() {}\n").unwrap();
fs::write(root.join("untouched.rs"), "fn retained_symbol() {}\n").unwrap();
fs::write(root.join("removed.rs"), "fn removed_symbol() {}\n").unwrap();
let (_, initial, _) = futures::executor::block_on(build_persisted_index(
root.clone(),
storage.path().to_path_buf(),
))
.unwrap();
publish_generation(storage.path(), &root, &initial.generation).unwrap();
// Keep the old reader open, as UI searches do during an update.
drop(initial.repository_lock);
fs::write(root.join("changed.rs"), "fn replacement_symbol() {}\n").unwrap();
fs::write(root.join("untouched.rs"), "fn must_not_be_read() {}\n").unwrap();
fs::remove_file(root.join("removed.rs")).unwrap();
fs::write(root.join("added.rs"), "fn added_symbol() {}\n").unwrap();
let paths = ["changed.rs", "removed.rs", "added.rs"]
.map(|path| root.join(path))
.into_iter()
.collect();
let (_, updated, count) = futures::executor::block_on(update_persisted_index(
root.clone(),
storage.path().to_path_buf(),
paths,
))
.unwrap();
assert_eq!(count, 3);
assert_eq!(updated.generation, initial.generation);
assert_eq!(updated.searcher.search_id("old_symbol").unwrap().len(), 0);
assert_eq!(
updated.searcher.search_id("removed_symbol").unwrap().len(),
0
);
for symbol in ["replacement_symbol", "retained_symbol", "added_symbol"] {
assert_eq!(updated.searcher.search_id(symbol).unwrap().len(), 1);
}
assert_eq!(
updated
.searcher
.search_id("must_not_be_read")
.unwrap()
.len(),
0
);
assert_eq!(initial.searcher.search_id("old_symbol").unwrap().len(), 1);
// A second writer can open while both readers are still alive; persisted contents survive reopen.
drop(updated.repository_lock);
let (_, reopened, count) = futures::executor::block_on(update_persisted_index(
root.clone(),
storage.path().to_path_buf(),
HashSet::from([root.join("added.rs")]),
))
.unwrap();
assert_eq!(count, 3);
assert_eq!(
reopened
.searcher
.search_id("replacement_symbol")
.unwrap()
.len(),
1
);
}
#[test]
fn indexed_body_is_capped_at_a_utf8_boundary_and_oversized_files_are_skipped() {
let root = TempDir::new().unwrap();
let path = root.path().join("unicode.txt");
let content = "€".repeat(MAX_INDEXED_BODY_BYTES);
fs::write(&path, &content).unwrap();
let document = build_document(&path).unwrap().unwrap();
assert_eq!(
document.body.len(),
MAX_INDEXED_BODY_BYTES - MAX_INDEXED_BODY_BYTES % 3
);
assert!(content.starts_with(&document.body));
let oversized = root.path().join("oversized.rs");
File::create(&oversized)
.unwrap()
.set_len(MAX_INDEXED_FILE_BYTES as u64 + 1)
.unwrap();
assert!(build_document(&oversized).unwrap().is_none());
let binary = root.path().join("binary.dat");
fs::write(&binary, [255, 254]).unwrap();
assert!(build_document(&binary).unwrap().is_none());
}
#[test]
fn subtree_updates_respect_inherited_ignores_and_remove_renamed_paths() {
let root = TempDir::new().unwrap();
let storage = TempDir::new().unwrap();
let root = dunce::canonicalize(root.path()).unwrap();
fs::create_dir(root.join("old")).unwrap();
fs::write(root.join(".gitignore"), "ignored.rs\n").unwrap();
fs::write(root.join("old/keep.rs"), "fn kept_symbol() {}\n").unwrap();
fs::write(root.join("old/ignored.rs"), "fn excluded_symbol() {}\n").unwrap();
let (_, initial, _) = futures::executor::block_on(build_persisted_index(
root.clone(),
storage.path().to_path_buf(),
))
.unwrap();
publish_generation(storage.path(), &root, &initial.generation).unwrap();
drop(initial);
fs::rename(root.join("old"), root.join("new")).unwrap();
let paths = HashSet::from([root.join("old"), root.join("new"), root.join("new/keep.rs")]);
let (_, updated, _) = futures::executor::block_on(update_persisted_index(
root.clone(),
storage.path().to_path_buf(),
paths,
))
.unwrap();
let hits = updated.searcher.search_id("kept_symbol").unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(
Path::new(&hits[0].values.file_path),
root.join("new/keep.rs")
);
assert_eq!(
updated.searcher.search_id("excluded_symbol").unwrap().len(),
0
);
}
crates/command-signatures-v2/build.rs
+4 -2
View File
@@ -8,6 +8,8 @@ fn main() -> anyhow::Result<()> {
println!("cargo:rerun-if-changed=js/build");
println!("cargo:rerun-if-changed=js/package.json");
println!("cargo:rerun-if-changed=js/tsconfig.json");
println!("cargo:rerun-if-changed=js/yarn.lock");
println!("cargo:rerun-if-changed=js/.yarnrc.yml");
if let Err(e) = build_command_signatures() {
if !Path::new(format!("{}/js/build", env!("CARGO_MANIFEST_DIR")).as_str()).exists() {
@@ -31,8 +33,8 @@ If you continue to encounter issues, ensure you don't have conflicting Node inst
}
fn build_command_signatures() -> anyhow::Result<()> {
match Command::new("yarn")
.arg("build")
match Command::new("corepack")
.args(["yarn", "build"])
.current_dir(format!("{}/js", env!("CARGO_MANIFEST_DIR")))
.output()
{
crates/galaxy_agent_core/src/provider_run.rs
+28 -27
View File
@@ -495,6 +495,7 @@ impl ProviderRun {
pub fn replace_tool_result_archive(&mut self, archive: Vec<ConversationMessage>) {
self.tool_result_archive = archive;
archive_tool_results(&mut self.tool_result_archive, Vec::new());
}
/// Replaces a summarized prefix while the run is parked between model calls.
@@ -782,6 +783,7 @@ impl ProviderRun {
&mut self,
recoverable_call_ids: &HashSet<String>,
) -> Result<ProviderRunRestoreNormalization, ProviderRunProtocolError> {
archive_tool_results(&mut self.tool_result_archive, Vec::new());
if matches!(self.state, ProviderRunState::AwaitingModel { .. }) {
self.state = ProviderRunState::Failed {
failure: ProviderRunFailure {
@@ -1740,43 +1742,42 @@ fn add_usage(total: &mut Usage, turn: &Usage) {
.saturating_add(turn.cache_creation_input_tokens);
}
fn archive_tool_results(
/// Retain tool history once per call/result ID, including when repairing older archives.
/// A multipart message is appended once, never once per tool contained in it.
pub fn archive_tool_results(
archive: &mut Vec<ConversationMessage>,
messages: Vec<ConversationMessage>,
) {
const MAX_TOOL_RESULT_ARCHIVE_ENTRIES: usize = 400;
let mut pending_tool_uses = Vec::new();
for message in messages {
match &message.content {
MessageContent::ToolUse { .. } => pending_tool_uses.push(message),
MessageContent::ToolResult { .. } => {
if let Some(tool_use) = pending_tool_uses.pop() {
archive.push(tool_use);
}
archive.push(message);
let mut seen_uses = HashSet::new();
let mut seen_results = HashSet::new();
let previous = std::mem::take(archive);
for mut message in previous.into_iter().chain(messages) {
let keep = match &mut message.content {
MessageContent::ToolUse { tool_use_id, .. } => seen_uses.insert(tool_use_id.clone()),
MessageContent::ToolResult { tool_use_id, .. } => {
seen_results.insert(tool_use_id.clone())
}
MessageContent::MultiPart(parts) => {
for part in parts {
match part {
ContentPart::ToolUse { .. } => pending_tool_uses.push(message.clone()),
ContentPart::ToolResult { .. } => {
if let Some(tool_use) = pending_tool_uses.pop() {
archive.push(tool_use);
}
archive.push(message.clone());
}
ContentPart::Text(_)
| ContentPart::Reasoning { .. }
| ContentPart::Image { .. } => {}
parts.retain(|part| match part {
ContentPart::ToolUse { tool_use_id, .. } => {
seen_uses.insert(tool_use_id.clone())
}
}
ContentPart::ToolResult { tool_use_id, .. } => {
seen_results.insert(tool_use_id.clone())
}
ContentPart::Text(_)
| ContentPart::Reasoning { .. }
| ContentPart::Image { .. } => false,
});
!parts.is_empty()
}
MessageContent::Text(_) => {}
MessageContent::Text(_) => false,
};
if keep {
archive.push(message);
}
}
archive.extend(pending_tool_uses);
if archive.len() > MAX_TOOL_RESULT_ARCHIVE_ENTRIES {
archive.drain(0..archive.len() - MAX_TOOL_RESULT_ARCHIVE_ENTRIES);
}
crates/galaxy_agent_core/src/provider_run_tests.rs
+39
View File
@@ -1426,3 +1426,42 @@ fn serialized_numeric_turn_budgets_remain_explicit_limits() {
assert_eq!(limits.max_model_turns, Some(100));
assert_eq!(ProviderRunLimits::default().max_model_turns, None);
}
#[test]
fn multipart_archive_is_not_multiplied_and_legacy_duplicates_are_repaired() {
let mut run = run();
let batch = accept_tool_turn(
&mut run,
tool_turn(
(0..4)
.map(|id| tool_call(&format!("search-{id}"), "search_codebase"))
.collect(),
&["search_codebase"],
),
);
for id in 0..4 {
run.complete_tool(
&batch.work_id,
successful_result(&format!("search-{id}"), &format!("file contents {id}")),
)
.unwrap();
}
run.commit_tool_batch(&batch.work_id).unwrap();
let transcript = run.transcript().to_vec();
let mut archive = Vec::new();
archive_tool_results(&mut archive, transcript.clone());
assert_eq!(archive.len(), 2);
for message in &archive {
let MessageContent::MultiPart(parts) = &message.content else {
panic!("expected multipart tools");
};
assert_eq!(parts.len(), 4);
}
let expected = archive.clone();
let duplicated: Vec<_> = (0..4).flat_map(|_| expected.clone()).collect();
run.replace_tool_result_archive(duplicated);
assert_eq!(run.tool_result_archive(), expected);
// A repeated compaction callback must not grow the archive either.
archive_tool_results(&mut archive, transcript);
assert_eq!(archive, expected);
}
crates/warp_search_core/src/searcher.rs
+48 -9
View File
@@ -470,19 +470,12 @@ impl SearcherWriterWrapper {
let guard = handle_copy.read();
let memory_budget = memory_budget.max(MIN_MEMORY_BUDGET);
let search_index = guard.search_index.clone();
let num_threads = std::cmp::min(
available_parallelism().map(|ap| ap.get()).unwrap_or(1),
MAX_THREADS_PER_INDEX_WRITER,
);
let writer = search_index
.writer_with_num_threads(num_threads, memory_budget)
.ok();
SearcherWriterWrapper {
search_index,
memory_budget,
composite_key_field,
writer,
writer: None,
reader_handle,
id_field_names: guard.id_fields.keys().cloned().collect(),
}
@@ -627,7 +620,13 @@ impl SearcherWriterWrapper {
fn writer(&mut self) -> anyhow::Result<&mut IndexWriter<TantivyDocument>> {
if self.writer.is_none() {
let writer = self.search_index.writer(self.memory_budget)?;
let num_threads = std::cmp::min(
available_parallelism().map(|ap| ap.get()).unwrap_or(1),
MAX_THREADS_PER_INDEX_WRITER.min(self.memory_budget / MIN_MEMORY_BUDGET),
);
let writer = self
.search_index
.writer_with_num_threads(num_threads, self.memory_budget)?;
self.writer = Some(writer);
}
@@ -785,6 +784,46 @@ impl<C: SearchSchemaConfig> SimpleFullTextSearcher<C> {
)
}
/// Atomically apply deletions and replacements in one commit, then release the writer.
/// Read-only handles remain usable without holding a disk writer lock or worker threads.
pub fn update_documents(
&self,
deleted: impl IntoIterator<Item = C::SearchIdEntry>,
documents: impl IntoIterator<Item = C::SearchDocEntry>,
) -> anyhow::Result<()> {
let mut writer = self.writer.lock();
let result = writer.execute_operations(
deleted
.into_iter()
.map(|entry| SearcherEvent::DocumentDeleted(entry.into_identifying_entry()))
.chain(
documents
.into_iter()
.map(|entry| SearcherEvent::DocumentInserted(entry.into_document_entry())),
),
);
let rollback = if result.is_err() {
writer
.writer
.as_mut()
.map(|active_writer| active_writer.rollback())
.transpose()
} else {
Ok(None)
};
// Let committed segment merges finish before releasing the disk lock. Dropping the
// writer immediately cancels merges and would accumulate small segments on every save.
let merges = writer
.writer
.take()
.map(|active_writer| active_writer.wait_merging_threads())
.transpose();
rollback?;
result?;
merges?;
Ok(())
}
#[allow(unused)]
/// Inserts a document into the index. It will overwrite any existing document with the same composite key hash.
pub fn insert_document(&self, entry: C::SearchDocEntry) -> anyhow::Result<()> {
crates/warp_search_core/src/searcher_tests.rs
+33
View File
@@ -310,3 +310,36 @@ fn test_searcher_async() {
"the index should be cleared and contain no documents"
);
}
#[test]
fn minimum_memory_budget_supports_repeated_atomic_updates() {
define_search_schema!(
schema_name: BUDGET_SCHEMA,
config_name: BudgetSchema,
search_doc: BudgetDocument,
identifying_doc: BudgetId,
search_fields: [name: 1.0],
id_fields: [id: u64]
);
let searcher = BUDGET_SCHEMA.create_searcher(MIN_MEMORY_BUDGET);
searcher
.update_documents(
Vec::<BudgetId>::new(),
[BudgetDocument {
id: 1,
name: "initial".into(),
}],
)
.unwrap();
searcher
.update_documents(
[BudgetId { id: 1 }],
[BudgetDocument {
id: 2,
name: "replacement".into(),
}],
)
.unwrap();
assert_eq!(searcher.search_id("initial").unwrap().len(), 0);
assert_eq!(searcher.search_id("replacement").unwrap().len(), 1);
}