//! Native crash reporting adapter that uses the [`minidumper`] crate with Sentry. This allows us //! to capture and report application crashes due to Unix signals like SIGSEGV (segfault) //! or Windows exceptions [https://learn.microsoft.com/en-us/windows/win32/debug/structured-exception-handling]. //! //! This is inspired by [`sentry-rust-minidump`](https://github.com/timfish/sentry-rust-minidump), //! with a few important changes: //! * Support for starting and stopping the crash-reporting process, since users can toggle crash reporting at runtime //! * Startup via our command-line parsing, rather than a separate hook //! * Use of anonymous, temporary crash dump files, to ensure they're cleaned up use std::{ collections::HashMap, fs::File, io::{self, Read as _, Seek as _, Write}, path::{Path, PathBuf}, process, sync::{ atomic::{AtomicBool, Ordering}, Arc, }, time::Duration, }; use anyhow::Context as _; use command::blocking::Command; use crash_handler::{CrashContext, CrashHandler}; use galaxy_core::report_error; use lazy_static::lazy_static; use parking_lot::Mutex; use sentry::{ protocol::{Attachment, AttachmentType}, Breadcrumb, Level, }; use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::ToSentryTags; lazy_static! { static ref GUARD: Mutex> = Mutex::new(None); } /// The minidump child process will exit if it doesn't receive a message after some time. This /// ensures that if the parent process exits without cleaning it up, the child won't linger /// forever. We ping the child every `PING_INTERVAL` to make sure it doesn't quit while the /// parent (this process) is running. const PING_INTERVAL: Duration = Duration::from_secs(5); /// Initialize the minidump reporter. pub fn init() { let mut global_guard = GUARD.lock(); match MinidumpGuard::start() { Ok(guard) => { *global_guard = Some(guard); } Err(err) => { report_error!(err); } } } /// Uninitialize the minidump reporter. pub fn uninit() { let maybe_guard = { GUARD.lock().take() }; // Ensure we drop the `MinidumpGuard` after releasing the GUARD mutex. If there's an // error stopping the server, we should log it as a Sentry breadcrumb in the Warp // process, but not forward the breadcrumb to the server process. std::mem::drop(maybe_guard); } /// Set a tag to include in minidump crash reports. pub fn set_tag(key: String, value: String) { let global_guard = GUARD.lock(); if let Some(guard) = global_guard.as_ref() { guard.set_tags(HashMap::from([(key, value)])); } } /// Set tags to include in minidump crash reports, using a type that implements [`ToSentryTags`]. pub fn set_tags_from(tags: &T) { let global_guard = GUARD.lock(); if let Some(guard) = global_guard.as_ref() { let tags = tags .to_sentry_tags() .into_iter() .map(|(k, v)| (k.to_string(), v)) .collect(); guard.set_tags(tags); } } /// Set the user id to include in minidump crash reports. pub fn set_user_id(user_id: &str) { let global_guard = GUARD.lock(); if let Some(guard) = global_guard.as_ref() { guard.set_user_id(user_id.to_owned()); } } /// Forward a breadcrumb to attach to minidump crash reports. pub fn forward_breadcrumb(breadcrumb: Breadcrumb) { let global_guard = GUARD.lock(); if let Some(guard) = global_guard.as_ref() { guard.add_breadcrumb(breadcrumb); } } /// Send a crash report via minidump. On certain platforms, this will produce an error report /// without actually crashing the process. pub fn crash() { let global_guard = GUARD.lock(); if let Some(guard) = global_guard.as_ref() { guard.crash(); } } /// Handle for minidump state that must be kept in scope while crash reporting is enabled. pub struct MinidumpGuard { child: process::Child, client: Arc, crash_handler: CrashHandler, } /// Run the minidump server process. pub fn run_server(socket_path: &Path) -> anyhow::Result<()> { // For troubleshooting, attempt to log from the minidump server. There's not much we can really // do if crash reporting fails, so creating the log file itself is best-effort. let log_dir = galaxy_core::paths::state_dir().join(galaxy_core::paths::WARP_LOGS_DIR); let _ = std::fs::create_dir_all(&log_dir); let log_path = log_dir.join("warp-minidump.log"); let log_target = File::create(log_path) .map(|file| env_logger::Target::Pipe(Box::new(file))) .unwrap_or_else(|_| env_logger::Target::Stdout); env_logger::builder() .parse_default_env() .target(log_target) .init(); let _guard = sentry::init(super::sentry_client_options()); struct Handler { shutdown: Arc, pending_crash_details: Mutex>, } impl minidumper::ServerHandler for Handler { fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> { // Use an anonymous temporary file for crash dumps. The path isn't used when writing a // dump, so we can use an empty value. let file = tempfile::tempfile()?; Ok((file, PathBuf::new())) } fn on_minidump_created( &self, result: Result, ) -> minidumper::LoopAction { if let Err(ref err) = &result { log::warn!("Unable to create minidump file: {err:#}"); } let crash_details = self.pending_crash_details.lock().take(); send_crash_report(crash_details, result.ok()); minidumper::LoopAction::Exit } fn on_message(&self, _kind: u32, buffer: Vec) { match bincode::deserialize::(&buffer) { Ok(MinidumpCommand::Shutdown) => { self.shutdown.store(true, Ordering::Relaxed); } Ok(MinidumpCommand::SetTags { tags }) => { sentry::configure_scope(|scope| { for (key, value) in tags { scope.set_tag(&key, value); } }); } Ok(MinidumpCommand::SetUser { user_id }) => { sentry::configure_scope(|scope| { scope.set_user(Some(sentry::User { id: Some(user_id), ..Default::default() })); }); } Ok(MinidumpCommand::AddBreadcrumb { breadcrumb }) => { sentry::add_breadcrumb(breadcrumb); } Ok(MinidumpCommand::SetCrashDetails { details }) => { *self.pending_crash_details.lock() = Some(details); } Err(err) => { log::warn!("Unable to deserialize minidump command: {err:#}"); } } } } let shutdown = Arc::new(AtomicBool::new(false)); let handler = Box::new(Handler { shutdown: shutdown.clone(), pending_crash_details: Default::default(), }); log::info!( "Starting minidump server listening on {}", socket_path.display() ); let result = minidumper::Server::with_name(socket_path) .context("Unable to create minidump server")? .run(handler, &shutdown, Some(2 * PING_INTERVAL)) .context("Error running minidump server"); if let Err(ref err) = result { log::error!("Error running minidump server: {err:#}"); } result } /// Uploads a crash report to Sentry, using the current scope. fn send_crash_report(details: Option, dump: Option) { let message = details.as_deref().unwrap_or("Fatal exception"); let crash_attachment = dump.and_then(|mut dump| { // In most cases, the minidump contents are available in memory. If not, we can read them off disk. let buffer = match dump.contents { Some(buffer) => buffer, None => { dump.file.flush().ok()?; dump.file.rewind().ok()?; let mut buffer = Vec::new(); dump.file.read_to_end(&mut buffer).ok()?; buffer } }; Some(Attachment { buffer, filename: "warp-minidump.dmp".to_string(), ty: Some(AttachmentType::Minidump), ..Default::default() }) }); sentry::with_scope( |scope| { // Do not use the crash reporting server for process info. scope.remove_extra("event.process"); if let Some(attachment) = crash_attachment { scope.add_attachment(attachment); } }, || sentry::capture_message(message, Level::Error), ); } impl MinidumpGuard { // NOTE: We CANNOT use `report_error`, `report_if_error`, `log`, or similar here. Those // all send information to Sentry, which can deadlock. /// Set up minidump-backed crash reporting. This spawns a child process that reports crashes to /// Sentry, and a crash handler which sends crashes to that child process. pub fn start() -> anyhow::Result { let socket_name = format!("wcr-{}.sock", Uuid::new_v4().simple()); let socket_path = if cfg!(target_os = "macos") { // On macOS, the maximum length of a socket path is fairly short, so use the temp directory. std::env::temp_dir().join(socket_name) } else { galaxy_core::paths::state_dir().join(socket_name) }; let child = Command::new(std::env::current_exe().context("Unable to get current executable path")?) .arg("minidump-server") .arg(&socket_path) .spawn() .context("Unable to spawn minidump server process")?; let client = Arc::new( wait_for_server(socket_path.as_path()).context("Unable to create minidump client")?, ); spawn_keepalive_thread(client.clone()); let client2 = client.clone(); let crash_handler = CrashHandler::attach(unsafe { crash_handler::make_crash_event(move |crash_context: &CrashContext| { if let Some(details) = format_crash_details(crash_context) { let _ = send_command( client.as_ref(), MinidumpCommand::SetCrashDetails { details }, ); } // Send a ping to the minidump server, ensuring that any messages sent before the // crash event are flushed and processed. This mostly only matters on macOS. let _ = client.ping(); let dump_result = client.request_dump(crash_context); crash_handler::CrashEventResult::Handled(dump_result.is_ok()) }) }) .context("Failed to attach crash signal handler")?; // Ensure that the crash server process can ptrace Warp. #[cfg(target_os = "linux")] crash_handler.set_ptracer(Some(child.id())); let guard = MinidumpGuard { child, client: client2, crash_handler, }; // Forward any existing tags to the minidump server. guard.set_tags(super::TAGS.read().clone()); Ok(guard) } /// Send the user id for the minidump server to attach to Sentry events. fn set_user_id(&self, user_id: String) { let _ = send_command(self.client.as_ref(), MinidumpCommand::SetUser { user_id }); } /// Send tags for the minidump server to attach to Sentry events. fn set_tags(&self, tags: HashMap) { let _ = send_command(self.client.as_ref(), MinidumpCommand::SetTags { tags }); } /// Add a breadcrumb to crash reports produced by the minidump server. fn add_breadcrumb(&self, breadcrumb: Breadcrumb) { let _ = send_command( self.client.as_ref(), MinidumpCommand::AddBreadcrumb { breadcrumb }, ); } /// Simulate a crash. pub fn crash(&self) { #[cfg(target_os = "linux")] self.crash_handler.simulate_signal(libc::SIGSEGV as _); #[cfg(not(target_os = "linux"))] self.crash_handler.simulate_exception(None); } } impl Drop for MinidumpGuard { fn drop(&mut self) { // Dropping the crash handler will detach it. // We can report errors here, as the minidump handler is no longer active. // Send a graceful shutdown command before killing the child process. if let Err(err) = send_command(&self.client, MinidumpCommand::Shutdown) { log::warn!("Unable to send shutdown command to minidump child process: {err:#}"); } if let Err(err) = self.child.kill() { log::warn!("Unable to kill minidump child process: {err:#}"); } } } /// Wait for the minidump server to start and return a client handle. /// /// Creating a [`minidumper::Client`] will fail unless the server has started. fn wait_for_server(socket_path: &Path) -> anyhow::Result { let start = instant::Instant::now(); let mut last_error = None; while start.elapsed() < Duration::from_secs(1) { match minidumper::Client::with_name(socket_path) { Ok(client) => { return Ok(client); } Err(err) => { last_error = Some(err); } } } match last_error { Some(err) => Err(err.into()), None => Err(anyhow::anyhow!("Unable to connect to minidump server")), } } /// Spawn a thread that periodically pings the minidump server to prevent it from idling out. fn spawn_keepalive_thread(client: Arc) { let _ = std::thread::Builder::new() .name("minidump-keepalive".to_string()) .spawn(move || loop { // Assume that if a ping fails, the server was shut down - the only purpose of this thread // is to prevent an idle timeout. if client.ping().is_err() { return; } std::thread::sleep(PING_INTERVAL); }); } /// Use `client` to send a command to the minidump server. fn send_command(client: &minidumper::Client, command: MinidumpCommand) -> anyhow::Result<()> { let message = bincode::serialize(&command).context("Failed to serialize minidump command")?; client .send_message(0, message) .context("Failed to send minidump command")?; Ok(()) } #[derive(Debug, Serialize, Deserialize)] enum MinidumpCommand { Shutdown, SetTags { tags: HashMap }, SetUser { user_id: String }, AddBreadcrumb { breadcrumb: Breadcrumb }, SetCrashDetails { details: String }, } /// Format details from a [`CrashContext`] into a Sentry error message. This information should /// already be in the minidump, but it's useful to surface prominently in Sentry. fn format_crash_details(crash_context: &CrashContext) -> Option { cfg_if::cfg_if! { if #[cfg(target_os = "linux")] { Some(format!("Killed by signal {} / {}", crash_context.siginfo.ssi_signo, crash_context.siginfo.ssi_code)) } else if #[cfg(target_os = "windows")] { Some(format!("Exception {}", crash_context.exception_code)) } else if #[cfg(target_os = "macos")] { crash_context.exception.as_ref().map(|exception| { format!("Exception {} ({} / {:?})", exception.kind, exception.code, exception.subcode) }) } else { None } } }