first pass of merging in warp (doesn't build)

app/src/server/telemetry/collector.rs

@@ -1,18 +1,90 @@
+use std::fs::remove_file;
 use std::sync::Arc;
+use std::time::Duration;
 
 use galaxyui::{Entity, ModelContext, SingletonEntity};
 
+use super::{clear_event_queue, rudder_event_file_path, RUDDER_TELEMETRY_EVENTS_FILE_NAME};
+use crate::auth::AuthStateProvider;
+use crate::channel::ChannelState;
+use crate::features::FeatureFlag;
 use crate::server::server_api::ServerApi;
+use crate::settings::{PrivacySettings, PrivacySettingsChangedEvent};
 
-/// No-op telemetry collector. Telemetry has been removed from Galaxy.
+// How often we send Active Usage signals.
+const ACTIVE_USAGE_DURATION: Duration = Duration::from_secs(60);
+
+/// Duration to wait before flushing the event queue to Rudderstack.
+const TELEMETRY_FLUSH_DURATION: Duration = Duration::from_secs(30);
+
+/// Max telemetry events to write to disk. This is bounded to limit the size of the file as well
+/// as latency of writing the file.
+const MAX_TELEMETRY_EVENTS_TO_STORE: usize = 20;
+
+/// Maximum time to wait for the telemetry flush network request during shutdown.
+/// If the network is unavailable or slow, we don't want the CLI process to hang indefinitely.
+const TELEMETRY_SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// App singleton responsible for scheduling periodic background tasks for sending batches of
+/// telemetry events to Rudderstack.  This model respects the user's telemetry enablement setting.
 pub struct TelemetryCollector {
     _server_api: Arc<ServerApi>,
 }
 
 impl TelemetryCollector {
     pub fn new(server_api: Arc<ServerApi>) -> Self {
-        Self {
-            _server_api: server_api,
+        Self { server_api }
+    }
+
+    pub fn initialize_telemetry_collection(&self, ctx: &mut ModelContext<TelemetryCollector>) {
+        // Start a background thread to periodically flush events from the telemetry event queue.
+        if ChannelState::is_release_bundle() || FeatureFlag::WithSandboxTelemetry.is_enabled() {
+            // Flush the events to Rudderstack that were persisted into a file the last time the app was
+            // quit.
+            self.flush_persisted_events_from_disk(ctx);
+        }
+
+        // Send Active App Usage signals
+        if FeatureFlag::RecordAppActiveEvents.is_enabled()
+            && (ChannelState::is_release_bundle() || FeatureFlag::WithSandboxTelemetry.is_enabled())
+        {
+            self.schedule_send_active_usage_event(ctx);
+        }
+
+        // Start a background thread to periodically flush events from the telemetry event queue.
+        if ChannelState::is_release_bundle()
+            || FeatureFlag::WithSandboxTelemetry.is_enabled()
+            || FeatureFlag::SendTelemetryToFile.is_enabled()
+        {
+            self.schedule_event_queue_flush(ctx);
+        }
+
+        // Clear queued telemetry events when telemetry is enabled or disabled. If telemetry is
+        // enabled, we will start sending Rudderstack requests when the event queue is periodically
+        // flushed. The initial request should not contain any events recorded when the user was
+        // previously opted-out of telemetry. In the case where the user turns the telemetry from
+        // on to off, we should not send another request with any telemetry, even if the event was
+        // initially recorded prior to the user turning telemetry off.`
+        ctx.subscribe_to_model(&PrivacySettings::handle(ctx), |_me, _, event, _ctx| {
+            if let PrivacySettingsChangedEvent::UpdateIsTelemetryEnabled { .. } = event {
+                clear_event_queue();
+            }
+        });
+    }
+
+    /// Writes all queued but unsent telemetry telemetry events to disk so that they may be sent
+    /// on the next app startup.
+    pub fn write_telemetry_events_to_disk(&self, ctx: &mut ModelContext<TelemetryCollector>) {
+        match self.server_api.persist_telemetry_events(
+            MAX_TELEMETRY_EVENTS_TO_STORE,
+            PrivacySettings::as_ref(ctx).get_snapshot(ctx),
+        ) {
+            Ok(()) => {
+                log::info!("Successfully wrote telemetry events to disk")
+            }
+            Err(e) => {
+                log::error!("Failed to write telemetry events to disk {e:#}");
+            }
         }
     }
 

app/src/server/telemetry/context.rs

@@ -1,5 +1,32 @@
-use serde_json::{json, Value};
+//! Module that builds a static context to attach to each of our events that are sent to Rudderstack.
+//! This is needed so we know the backing operating system and version of each telemetry event.
 
+use std::sync::OnceLock;
+
+use serde::Serialize;
+use serde_json::{json, Value};
+#[cfg(target_family = "wasm")]
+use galaxyui::platform::wasm;
+
+use super::rudder_message::Message as RudderMessage;
+use crate::server::OperatingSystemInfo;
+
+static TELEMETRY_CONTEXT: OnceLock<TelemetryContext> = OnceLock::new();
+
+#[derive(Serialize)]
+struct TelemetryContextInfo {
+    /// Info about the operating system of the client.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    os: Option<&'static OperatingSystemInfo>,
+    /// The user agent provided by the browser, if running on Web. If not on
+    /// Web, this is always `None`.
+    #[serde(rename = "userAgent", skip_serializing_if = "Option::is_none")]
+    user_agent: Option<String>,
+}
+
+/// Newtype representing a [`Value`] with a serialized version of the context that we send to
+/// Rudderstack.
+/// See https://www.rudderstack.com/docs/event-spec/standard-events/common-fields/#contextual-fields.
 pub struct TelemetryContext(Value);
 
 impl TelemetryContext {
@@ -9,7 +36,6 @@ impl TelemetryContext {
 }
 
 pub fn telemetry_context() -> &'static TelemetryContext {
-    use std::sync::OnceLock;
     static TELEMETRY_CONTEXT: OnceLock<TelemetryContext> = OnceLock::new();
     TELEMETRY_CONTEXT.get_or_init(|| TelemetryContext(json!({})))
 }

app/src/server/telemetry/context_provider.rs

@@ -24,29 +24,3 @@ impl TelemetryContextProvider for AppTelemetryContextProvider {
         auth_state.anonymous_id()
     }
 }
-
-/// A no-op telemetry context provider for headless contexts (e.g. the remote
-/// server daemon) that run without authentication. Telemetry events that
-/// require a user/anonymous ID will silently produce empty identifiers,
-/// preventing panics from an unregistered `TelemetryContextModel` singleton.
-#[cfg_attr(target_family = "wasm", allow(dead_code))]
-pub struct NoopTelemetryContextProvider;
-
-impl NoopTelemetryContextProvider {
-    #[cfg_attr(target_family = "wasm", allow(dead_code))]
-    pub fn new_context_provider(
-        _ctx: &mut ModelContext<TelemetryContextModel>,
-    ) -> TelemetryContextModel {
-        Box::new(Self)
-    }
-}
-
-impl TelemetryContextProvider for NoopTelemetryContextProvider {
-    fn user_id(&self, _ctx: &AppContext) -> Option<String> {
-        None
-    }
-
-    fn anonymous_id(&self, _ctx: &AppContext) -> String {
-        String::new()
-    }
-}

app/src/server/telemetry/mod.rs

@@ -6,11 +6,47 @@ mod macros;
 pub mod rudder_message;
 pub mod secret_redaction;
 
+use std::fs::File;
+#[cfg(not(target_family = "wasm"))]
+use std::fs::OpenOptions;
+use std::future::Future;
+use std::path::{Path, PathBuf};
+
+use anyhow::Result;
+use chrono::Utc;
 pub use collector::*;
 pub use context::telemetry_context;
 pub use events::*;
+use futures::FutureExt;
+use rudder_message::{
+    Batch as RudderBatch, BatchMessage as RudderBatchMessageWithMetadata,
+    BatchMessageItem as RudderBatchMessage, Message as RudderMessage,
+};
+use galaxy_core::channel::RudderStackDestination;
+use warpui::telemetry::Event;
+
+use crate::auth::UserUid;
+use crate::features::FeatureFlag;
+use crate::server::telemetry::context::AttachContext;
+use crate::server::telemetry_ext::TelemetryExt;
+use crate::settings::PrivacySettingsSnapshot;
+use crate::ChannelState;
+
+/// Filename for file where telemetry events are written on app quit.
+const RUDDER_TELEMETRY_EVENTS_FILE_NAME: &str = "rudder_telemetry_events.json";
+
+/// Filepath where the Rudder events should be written on app quit.
+fn rudder_event_file_path() -> PathBuf {
+    galaxy_core::paths::secure_state_dir()
+        .unwrap_or_else(galaxy_core::paths::state_dir)
+        .join(RUDDER_TELEMETRY_EVENTS_FILE_NAME)
+}
+
+/// Removes all telemetry events from the app telemetry event queue.
+pub fn clear_event_queue() {
+    let _ = galaxyui::telemetry::flush_events();
+}
 
-/// No-op stub. Telemetry has been removed from Galaxy.
 pub struct TelemetryApi {
     pub client: http_client::Client,
 }
@@ -30,11 +66,166 @@ impl TelemetryApi {
 
     pub async fn send_telemetry_event(
         &self,
-        _user_id: Option<crate::auth::UserUid>,
-        _anonymous_id: String,
-        _event: impl galaxy_core::telemetry::TelemetryEvent,
-        _settings_snapshot: crate::settings::PrivacySettingsSnapshot,
-    ) -> anyhow::Result<()> {
+        user_id: Option<UserUid>,
+        anonymous_id: String,
+        event: impl galaxy_core::telemetry::TelemetryEvent,
+        settings_snapshot: PrivacySettingsSnapshot,
+    ) -> Result<()> {
+        let event = galaxyui::telemetry::create_event(
+            user_id.map(|uid| uid.as_string()),
+            anonymous_id,
+            event.name().into(),
+            event.payload(),
+            event.contains_ugc(),
+            galaxyui::time::get_current_time(),
+        );
+
+        self.send_telemetry_event_internal(event, settings_snapshot)
+            .await
+    }
+
+    /// Internal implementation for sending telemetry events. This reduces code size, since
+    // we:
+    // 1. Return a boxed future, so calling `async` functions don't need to inline this one.
+    // 2. Don't have to monomorphize for each telemetry event implementation.
+    fn send_telemetry_event_internal(
+        &self,
+        event: Event,
+        settings_snapshot: PrivacySettingsSnapshot,
+    ) -> impl Future<Output = Result<()>> + '_ {
+        let work = async move {
+            if settings_snapshot.should_disable_telemetry() {
+                log::info!("Not sending telemetry event because telemetry is disabled.");
+                return Result::Ok(());
+            }
+
+            #[cfg(not(target_family = "wasm"))]
+            if FeatureFlag::SendTelemetryToFile.is_enabled() {
+                self.persist_events_to_telemetry_log_file(vec![event.clone()])?;
+            }
+
+            if !(ChannelState::is_release_bundle()
+                || FeatureFlag::WithSandboxTelemetry.is_enabled())
+            {
+                return Result::Ok(());
+            }
+
+            let rudder_batch = vec![event.to_rudder_batch_message()];
+
+            let result = self
+                .send_batch_messages_to_rudder(rudder_batch, settings_snapshot)
+                .await;
+
+            // This is only conditionally compiled because `is_connect` is not
+            // available on wasm.  If additional checks are made against the
+            // `reqwest::Error`, this condition should be performed specifically
+            // against `is_connect` and not the whole loop.
+            #[cfg(not(target_family = "wasm"))]
+            if let Err(error) = &result {
+                for cause in error.chain() {
+                    if let Some(err) = cause.downcast_ref::<reqwest::Error>() {
+                        if err.is_connect() {
+                            log::warn!("Failed to send telemetry event: {error}");
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+
+            result
+        };
+
+        // On WASM, the work future is non-Send, because the HTTP request future contains a reference to a JS
+        // value (which is fine, since our WASM executor is single-threaded). On all other platforms, we must
+        // return a Send future in order to use the background executor.
+        cfg_if::cfg_if! {
+            if #[cfg(target_family = "wasm")] {
+                work.boxed_local()
+            } else {
+                work.boxed()
+            }
+        }
+    }
+
+    /// Send a batch of RudderStack messages to their HTTP API.
+    /// Note that the rudderanalytics SDK provides a client, but we don't
+    /// use it for a few reasons:
+    /// 1. It only supports a blocking HTTP client instead of an async one
+    /// 2. We want to use our own HTTP client which has before/after request logging hooks
+    #[cfg_attr(target_family = "wasm", allow(clippy::question_mark))]
+    async fn send_batch_messages_to_rudder(
+        &self,
+        messages: Vec<RudderBatchMessageWithMetadata>,
+        settings_snapshot: PrivacySettingsSnapshot,
+    ) -> Result<()> {
+        if messages.is_empty() {
+            log::debug!("Dropping empty RudderStack telemetry batch");
+            return Ok(());
+        }
+
+        if settings_snapshot.should_disable_telemetry() {
+            log::info!("Not sending batched messages because telemetry is disabled.");
+            return Ok(());
+        }
+
+        log::info!("Start to send telemetry events to RudderStack");
+
+        let (mut messages_with_ugc, messages_without_ugc): (Vec<_>, Vec<_>) = messages
+            .into_iter()
+            .partition(|message| message.contains_ugc);
+
+        // If we shouldn't collect UGC telemetry, forcibly clear any messages with UGC before trying to send.
+        if !settings_snapshot.should_collect_ai_ugc_telemetry() {
+            messages_with_ugc.clear();
+        }
+
+        for (messages, rudder_stack_destination) in [
+            (
+                messages_with_ugc,
+                ChannelState::rudderstack_ugc_destination(),
+            ),
+            (
+                messages_without_ugc,
+                ChannelState::rudderstack_non_ugc_destination(),
+            ),
+        ] {
+            if messages.is_empty() {
+                continue;
+            }
+
+            // Note that timestamp and context are already included in the individual RudderBatchMessages
+            // and these are the most important ones,
+            // but we also add them to the RudderMessage::Batch wrapper.
+            let rudder_message = RudderMessage::Batch(RudderBatch {
+                batch: messages
+                    .into_iter()
+                    .map(|message| message.message)
+                    .collect(),
+                original_timestamp: Some(Utc::now()),
+                ..Default::default()
+            });
+            if let Err(e) = self
+                .send_rudder_request(rudder_message, rudder_stack_destination)
+                .await
+            {
+                // Don't treat a connection issue as an error as these are outside of our control.
+                //
+                // This is only conditionally compiled because `is_connect` is not
+                // available on wasm.  If additional checks are made against the
+                // `reqwest::Error`, this condition should be performed specifically
+                // against `is_connect` and not the whole loop.
+                #[cfg(not(target_family = "wasm"))]
+                for cause in e.chain() {
+                    if let Some(err) = cause.downcast_ref::<reqwest::Error>() {
+                        if err.is_connect() {
+                            log::warn!("Failed to send event to RudderStack: {e}");
+                            return Ok(());
+                        }
+                    }
+                }
+                return Err(e);
+            }
+        }
         Ok(())
     }
 

app/src/server/telemetry/secret_redaction.rs

@@ -1,7 +1,35 @@
-use regex::Regex;
+//! Best-effort secret redaction for telemetry payloads.
+//!
+//! Unlike the AI-side secret redaction in `app/src/ai/blocklist/block/secret_redaction.rs`,
+//! which is gated on the user's secret-redaction (a.k.a. "safe mode") setting and is used
+//! for visual obfuscation in the terminal, the redaction in this module is unconditional:
+//! we always do a redaction pass on telemetry payloads that may contain user-generated
+//! content, regardless of the user's safe-mode setting. The two settings are deliberately
+//! decoupled — visual obfuscation is a UX preference, while telemetry-side redaction is a
+//! defence-in-depth measure for data leaving the device.
+//!
+//! The regex used for redaction always includes the default patterns defined in
+//! `crate::terminal::model::secrets::regexes::DEFAULT_REGEXES_WITH_NAMES`. Any custom
+//! patterns the user has configured (or that their organization has configured via
+//! enterprise secret redaction) are layered on top of those defaults.
+//!
+//! This module is intentionally lightweight: it does byte-range matching only and does
+//! not track `SecretLevel`s or character ranges, since the telemetry path doesn't need
+//! either.
+use std::collections::HashSet;
+use std::ops::Range;
 
-pub fn update_telemetry_secrets_regex<'a>(
-    _user_secrets: impl Iterator<Item = &'a Regex>,
-    _enterprise_secrets: impl Iterator<Item = &'a Regex>,
-) {
+use lazy_static::lazy_static;
+use parking_lot::RwLock;
+use regex_automata::meta::Regex;
+use serde_json::Value;
+
+use crate::terminal::model::secrets::regexes::DEFAULT_REGEXES_WITH_NAMES;
+const REDACTION_REPLACEMENT_CHARACTER: &str = "*";
+lazy_static! {
+    /// Regex used to redact secrets from telemetry payloads. Initialized with the
+    /// default patterns so that redaction works even before the user's privacy
+    /// settings are loaded (and even for users who have never configured any
+    /// custom patterns).
+    static ref TELEMETRY_SECRETS_REGEX: RwLock<Regex> = RwLock::new(build_default_regex());
 }