Rebrand to Galaxy, major improvements to Bedrock support, still needs some TLC though
This commit is contained in:
Ryan Ward
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "galaxy_ripgrep"
|
||||
authors = ["Warp Team <dev@warp.dev>"]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Thin wrapper around ripgrep for usage by Warp CLI"
|
||||
publish.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
grep = "0.4.1"
|
||||
ignore = "0.4.20"
|
||||
instant.workspace = true
|
||||
log.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
string-offset.workspace = true
|
||||
galaxy_cli.workspace = true
|
||||
|
||||
[target.'cfg(not(target_family = "wasm"))'.dependencies]
|
||||
async-process.workspace = true
|
||||
command.workspace = true
|
||||
futures.workspace = true
|
||||
|
||||
[target.'cfg(target_family = "wasm")'.dependencies]
|
||||
getrandom = { version = "0.2", features = ["js"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix.workspace = true
|
||||
@@ -0,0 +1,21 @@
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: {} <pattern> [paths...]", args[0]);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let pattern = args[1].clone();
|
||||
let paths: Vec<PathBuf> = args[2..].iter().map(PathBuf::from).collect();
|
||||
|
||||
println!("Searching for pattern: {pattern}");
|
||||
println!("In paths: {paths:?}");
|
||||
|
||||
galaxy_ripgrep::search::run_search_subprocess(&[pattern], paths, false, false, None)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//! Thin wrapper around ripgrep for searching files.
|
||||
|
||||
pub mod search;
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
mod types;
|
||||
|
||||
/// On Unix, monitor the parent PID and exit this process if it changes.
|
||||
///
|
||||
/// This is used by the ripgrep worker process to ensure we don't keep
|
||||
/// searching if the main Warp process has exited.
|
||||
#[cfg(unix)]
|
||||
pub fn monitor_parent_and_exit_on_change(parent_pid: Option<u32>) {
|
||||
use nix::unistd::Pid;
|
||||
use std::{thread, time::Duration};
|
||||
|
||||
let expected_parent = match parent_pid {
|
||||
Some(pid) => Pid::from_raw(pid as i32),
|
||||
None => return,
|
||||
};
|
||||
|
||||
thread::spawn(move || {
|
||||
loop {
|
||||
// If we've been reparented to a different process, the original
|
||||
// parent died so we should exit.
|
||||
if Pid::parent() != expected_parent {
|
||||
log::info!(
|
||||
"ripgrep helper: detected parent pid change (expected {expected_parent}); \
|
||||
exiting search process."
|
||||
);
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use grep::regex::RegexMatcherBuilder;
|
||||
use grep::{
|
||||
printer::JSONBuilder,
|
||||
searcher::{BinaryDetection, SearcherBuilder},
|
||||
};
|
||||
use ignore::{WalkBuilder, WalkState};
|
||||
use std::ops::Not;
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
/// Maximum line length (in bytes) the ripgrep searcher will tolerate.
|
||||
/// Files containing lines longer than this are skipped, preventing
|
||||
/// unbounded memory growth from minified or generated files.
|
||||
/// Only applied in single-line mode; multiline search needs the full file
|
||||
/// in memory so the limit would be per-file rather than per-line.
|
||||
const SEARCHER_LINE_HEAP_LIMIT: usize = 64 * 1024;
|
||||
|
||||
/// A single submatch span within a matched line.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Submatch {
|
||||
/// Byte offset into `line_text` where this submatch starts.
|
||||
pub byte_start: ByteOffset,
|
||||
/// Byte offset into `line_text` where this submatch ends (exclusive).
|
||||
pub byte_end: ByteOffset,
|
||||
}
|
||||
|
||||
/// A single search match: one line in one file, with submatch highlights.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Match {
|
||||
pub file_path: PathBuf,
|
||||
pub line_number: u32,
|
||||
pub line_text: String,
|
||||
pub submatches: Vec<Submatch>,
|
||||
}
|
||||
|
||||
/// Entry point for the ripgrep subprocess.
|
||||
///
|
||||
/// Runs a ripgrep search in-process and writes JSON results to stdout.
|
||||
/// The main Warp process spawns this via the `ripgrep-search` CLI
|
||||
/// subcommand and reads the JSON output.
|
||||
pub fn run_search_subprocess(
|
||||
patterns: &[String],
|
||||
paths: Vec<PathBuf>,
|
||||
ignore_case: bool,
|
||||
multiline: bool,
|
||||
#[cfg_attr(not(unix), allow(unused_variables))] parent_pid: Option<u32>,
|
||||
) -> anyhow::Result<()> {
|
||||
#[cfg(unix)]
|
||||
crate::monitor_parent_and_exit_on_change(parent_pid);
|
||||
|
||||
if patterns.is_empty() {
|
||||
return Err(anyhow!("No patterns specified"));
|
||||
}
|
||||
if paths.is_empty() {
|
||||
return Err(anyhow!("No paths specified"));
|
||||
}
|
||||
|
||||
let mut matcher_builder = RegexMatcherBuilder::new();
|
||||
matcher_builder.case_insensitive(ignore_case);
|
||||
if multiline {
|
||||
matcher_builder.line_terminator(None);
|
||||
}
|
||||
let matcher = matcher_builder.build_many(patterns)?;
|
||||
|
||||
let stdout = std::sync::Mutex::new(std::io::stdout());
|
||||
|
||||
let mut walker_builder = WalkBuilder::new(&paths[0]);
|
||||
for path in paths.iter().skip(1) {
|
||||
walker_builder.add(path);
|
||||
}
|
||||
let walker = walker_builder.build_parallel();
|
||||
|
||||
walker.run(|| {
|
||||
let matcher = matcher.clone();
|
||||
let stdout = &stdout;
|
||||
|
||||
// Allocate once per thread and reuse across entries.
|
||||
let mut buf = Vec::new();
|
||||
let mut searcher = SearcherBuilder::new()
|
||||
.binary_detection(BinaryDetection::quit(b'\x00'))
|
||||
.line_number(true)
|
||||
.multi_line(multiline)
|
||||
.heap_limit(multiline.not().then_some(SEARCHER_LINE_HEAP_LIMIT))
|
||||
.build();
|
||||
|
||||
Box::new(move |entry| {
|
||||
let entry = match entry {
|
||||
Ok(entry) => entry,
|
||||
Err(err) => {
|
||||
log::warn!("ripgrep walk error: {err}");
|
||||
return WalkState::Continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !entry.file_type().is_some_and(|ft| ft.is_file()) {
|
||||
return WalkState::Continue;
|
||||
}
|
||||
|
||||
// Search into the thread-local buffer, then flush the
|
||||
// complete JSON lines to stdout under a lock so that
|
||||
// output from parallel threads never interleaves.
|
||||
buf.clear();
|
||||
let mut printer = JSONBuilder::new().build(std::io::Cursor::new(&mut buf));
|
||||
|
||||
if let Err(err) = searcher.search_path(
|
||||
&matcher,
|
||||
entry.path(),
|
||||
printer.sink_with_path(&matcher, entry.path()),
|
||||
) {
|
||||
log::warn!(
|
||||
"ripgrep search error for {}: {}",
|
||||
entry.path().display(),
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
if !buf.is_empty() {
|
||||
let Ok(mut out) = stdout.lock() else {
|
||||
// Mutex poisoned — another search thread panicked.
|
||||
return WalkState::Quit;
|
||||
};
|
||||
use std::io::Write;
|
||||
if out.write_all(&buf).is_err() {
|
||||
// Stdout pipe is broken (parent likely cancelled the
|
||||
// search), so stop walking.
|
||||
return WalkState::Quit;
|
||||
}
|
||||
}
|
||||
|
||||
WalkState::Continue
|
||||
})
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
mod process_impl {
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
|
||||
use futures::io::{AsyncBufReadExt as _, BufReader};
|
||||
use futures::stream::Stream;
|
||||
use futures::StreamExt as _;
|
||||
|
||||
use super::{Match, Submatch};
|
||||
use crate::types::RipgrepMessage;
|
||||
|
||||
/// Searches `paths` for lines matching `patterns` and returns all results.
|
||||
///
|
||||
/// Spawns a child process that runs the ripgrep search, collects every
|
||||
/// match, and returns them once the search is complete.
|
||||
///
|
||||
/// For incremental results, use [`search_streaming`] instead.
|
||||
pub async fn search(
|
||||
patterns: &[String],
|
||||
paths: &[PathBuf],
|
||||
ignore_case: bool,
|
||||
multiline: bool,
|
||||
) -> anyhow::Result<Vec<Match>> {
|
||||
let stream = search_streaming(patterns, paths, ignore_case, multiline)?;
|
||||
Ok(stream.collect().await)
|
||||
}
|
||||
|
||||
/// Searches `paths` for lines matching `patterns`, returning a stream
|
||||
/// of matches as they are found.
|
||||
///
|
||||
/// This is the preferred entry point when responsiveness matters (e.g.
|
||||
/// the global search UI). The caller controls batching and throttling.
|
||||
pub fn search_streaming(
|
||||
patterns: &[String],
|
||||
paths: &[PathBuf],
|
||||
ignore_case: bool,
|
||||
multiline: bool,
|
||||
) -> anyhow::Result<impl Stream<Item = Match>> {
|
||||
let child = spawn_search_process(patterns, paths, ignore_case, multiline)?;
|
||||
Ok(match_stream_from_child(child))
|
||||
}
|
||||
|
||||
/// Spawns the warp CLI with the `ripgrep-search` subcommand.
|
||||
fn spawn_search_process(
|
||||
patterns: &[String],
|
||||
paths: &[PathBuf],
|
||||
ignore_case: bool,
|
||||
multiline: bool,
|
||||
) -> Result<async_process::Child, std::io::Error> {
|
||||
let current_exe = std::env::current_exe()?;
|
||||
let mut cmd = command::r#async::Command::new(current_exe);
|
||||
|
||||
cmd.arg(galaxy_cli::ripgrep_search_subcommand())
|
||||
.arg(galaxy_cli::parent_flag());
|
||||
|
||||
if ignore_case {
|
||||
cmd.arg("--ignore-case");
|
||||
}
|
||||
|
||||
if multiline {
|
||||
cmd.arg("--multiline");
|
||||
}
|
||||
|
||||
for pattern in patterns {
|
||||
cmd.arg(pattern);
|
||||
}
|
||||
|
||||
for path in paths {
|
||||
cmd.arg(path);
|
||||
}
|
||||
|
||||
cmd.kill_on_drop(true)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped());
|
||||
|
||||
cmd.spawn()
|
||||
}
|
||||
|
||||
/// Turns a child process (with piped stdout) into a stream of parsed matches.
|
||||
fn match_stream_from_child(mut child: async_process::Child) -> impl Stream<Item = Match> {
|
||||
let stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.expect("spawn_search_process must pipe stdout");
|
||||
let reader = BufReader::new(stdout);
|
||||
|
||||
futures::stream::unfold((reader, child), |(mut reader, child)| async move {
|
||||
let mut line = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) | Err(_) => return None,
|
||||
Ok(_) => {}
|
||||
}
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<RipgrepMessage>(&line) {
|
||||
Ok(RipgrepMessage::Match { data }) => {
|
||||
let submatches = data
|
||||
.submatches
|
||||
.into_iter()
|
||||
.map(|s| Submatch {
|
||||
byte_start: s.start,
|
||||
byte_end: s.end,
|
||||
})
|
||||
.collect();
|
||||
let m = Match {
|
||||
file_path: PathBuf::from(data.path.text),
|
||||
line_number: data.line_number,
|
||||
line_text: data.lines.text,
|
||||
submatches,
|
||||
};
|
||||
return Some((m, (reader, child)));
|
||||
}
|
||||
Ok(RipgrepMessage::Begin | RipgrepMessage::End) => continue,
|
||||
Err(err) => {
|
||||
log::warn!("ripgrep: failed to parse JSON line: {err}");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub use process_impl::{search, search_streaming};
|
||||
@@ -0,0 +1,38 @@
|
||||
//! Deserialization types for ripgrep JSON output.
|
||||
use string_offset::ByteOffset;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct RipgrepPath {
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct RipgrepLines {
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct RipgrepSubmatch {
|
||||
pub start: ByteOffset,
|
||||
pub end: ByteOffset,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct RipgrepMatchData {
|
||||
pub path: RipgrepPath,
|
||||
pub lines: RipgrepLines,
|
||||
pub line_number: u32,
|
||||
#[serde(default)]
|
||||
pub submatches: Vec<RipgrepSubmatch>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "lowercase")]
|
||||
pub(crate) enum RipgrepMessage {
|
||||
#[serde(rename = "begin")]
|
||||
Begin,
|
||||
#[serde(rename = "match")]
|
||||
Match { data: RipgrepMatchData },
|
||||
#[serde(rename = "end")]
|
||||
End,
|
||||
}
|
||||
Reference in New Issue
Block a user