Reduce indexing overhead and repair agent search and child recovery

crates/warp_search_core/src/searcher.rs

@@ -470,19 +470,12 @@ impl SearcherWriterWrapper {
         let guard = handle_copy.read();
         let memory_budget = memory_budget.max(MIN_MEMORY_BUDGET);
         let search_index = guard.search_index.clone();
-        let num_threads = std::cmp::min(
-            available_parallelism().map(|ap| ap.get()).unwrap_or(1),
-            MAX_THREADS_PER_INDEX_WRITER,
-        );
-        let writer = search_index
-            .writer_with_num_threads(num_threads, memory_budget)
-            .ok();
 
         SearcherWriterWrapper {
             search_index,
             memory_budget,
             composite_key_field,
-            writer,
+            writer: None,
             reader_handle,
             id_field_names: guard.id_fields.keys().cloned().collect(),
         }
@@ -627,7 +620,13 @@ impl SearcherWriterWrapper {
 
     fn writer(&mut self) -> anyhow::Result<&mut IndexWriter<TantivyDocument>> {
         if self.writer.is_none() {
-            let writer = self.search_index.writer(self.memory_budget)?;
+            let num_threads = std::cmp::min(
+                available_parallelism().map(|ap| ap.get()).unwrap_or(1),
+                MAX_THREADS_PER_INDEX_WRITER.min(self.memory_budget / MIN_MEMORY_BUDGET),
+            );
+            let writer = self
+                .search_index
+                .writer_with_num_threads(num_threads, self.memory_budget)?;
             self.writer = Some(writer);
         }
 
@@ -785,6 +784,46 @@ impl<C: SearchSchemaConfig> SimpleFullTextSearcher<C> {
         )
     }
 
+    /// Atomically apply deletions and replacements in one commit, then release the writer.
+    /// Read-only handles remain usable without holding a disk writer lock or worker threads.
+    pub fn update_documents(
+        &self,
+        deleted: impl IntoIterator<Item = C::SearchIdEntry>,
+        documents: impl IntoIterator<Item = C::SearchDocEntry>,
+    ) -> anyhow::Result<()> {
+        let mut writer = self.writer.lock();
+        let result = writer.execute_operations(
+            deleted
+                .into_iter()
+                .map(|entry| SearcherEvent::DocumentDeleted(entry.into_identifying_entry()))
+                .chain(
+                    documents
+                        .into_iter()
+                        .map(|entry| SearcherEvent::DocumentInserted(entry.into_document_entry())),
+                ),
+        );
+        let rollback = if result.is_err() {
+            writer
+                .writer
+                .as_mut()
+                .map(|active_writer| active_writer.rollback())
+                .transpose()
+        } else {
+            Ok(None)
+        };
+        // Let committed segment merges finish before releasing the disk lock. Dropping the
+        // writer immediately cancels merges and would accumulate small segments on every save.
+        let merges = writer
+            .writer
+            .take()
+            .map(|active_writer| active_writer.wait_merging_threads())
+            .transpose();
+        rollback?;
+        result?;
+        merges?;
+        Ok(())
+    }
+
     #[allow(unused)]
     /// Inserts a document into the index. It will overwrite any existing document with the same composite key hash.
     pub fn insert_document(&self, entry: C::SearchDocEntry) -> anyhow::Result<()> {

crates/warp_search_core/src/searcher_tests.rs

@@ -310,3 +310,36 @@ fn test_searcher_async() {
         "the index should be cleared and contain no documents"
     );
 }
+
+#[test]
+fn minimum_memory_budget_supports_repeated_atomic_updates() {
+    define_search_schema!(
+        schema_name: BUDGET_SCHEMA,
+        config_name: BudgetSchema,
+        search_doc: BudgetDocument,
+        identifying_doc: BudgetId,
+        search_fields: [name: 1.0],
+        id_fields: [id: u64]
+    );
+    let searcher = BUDGET_SCHEMA.create_searcher(MIN_MEMORY_BUDGET);
+    searcher
+        .update_documents(
+            Vec::<BudgetId>::new(),
+            [BudgetDocument {
+                id: 1,
+                name: "initial".into(),
+            }],
+        )
+        .unwrap();
+    searcher
+        .update_documents(
+            [BudgetId { id: 1 }],
+            [BudgetDocument {
+                id: 2,
+                name: "replacement".into(),
+            }],
+        )
+        .unwrap();
+    assert_eq!(searcher.search_id("initial").unwrap().len(), 0);
+    assert_eq!(searcher.search_id("replacement").unwrap().len(), 1);
+}