Initial public release of Warp.

Repo-Sync-Origin: warpdotdev/warp-internal@12af1d983b

app/src/experiments/block_onboarding_layer.rs

@@ -0,0 +1,64 @@
+use super::Experiment;
+use crate::experiments::{BucketRange, Layer};
+use lazy_static::lazy_static;
+use std::{collections::HashMap, str::FromStr};
+
+lazy_static! {
+    pub static ref BLOCK_ONBOARDING_LAYER: Layer = Layer {
+        name: "BlockOnboardingLayer",
+        hasher_seeds: (2222, 9999),
+        traffic_allocations: HashMap::from([
+            (BlockOnboarding::VariantOne.get_group_id(), 30.0),
+            (BlockOnboarding::VariantTwo.get_group_id(), 70.0)
+        ]),
+        bucket_ranges: vec![
+            BucketRange::new(BlockOnboarding::VariantTwo, 0..333),
+            BucketRange::new(BlockOnboarding::VariantOne, 333..633),
+            BucketRange::new(BlockOnboarding::VariantTwo, 633..1000),
+        ]
+    };
+}
+
+/// An experiment to test block onboarding's impact on user activation.
+#[derive(Debug)]
+pub enum BlockOnboarding {
+    /// No onboarding survey, just theme + prompt. No welcome block (ascii)
+    VariantOne,
+    /// Onboarding survey and theme + prompt. No welcome block (ascii)
+    VariantTwo,
+}
+
+const BLOCK_ONBOARDING_VARIANT_ONE: &str = "VariantOne";
+const BLOCK_ONBOARDING_VARIANT_TWO: &str = "VariantTwo";
+
+impl Experiment<BlockOnboarding> for BlockOnboarding {
+    fn name() -> &'static str {
+        "BlockOnboarding"
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            BlockOnboarding::VariantOne => BLOCK_ONBOARDING_VARIANT_ONE,
+            BlockOnboarding::VariantTwo => BLOCK_ONBOARDING_VARIANT_TWO,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        false
+    }
+}
+
+impl FromStr for BlockOnboarding {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            BLOCK_ONBOARDING_VARIANT_ONE => Ok(BlockOnboarding::VariantOne),
+            BLOCK_ONBOARDING_VARIANT_TWO => Ok(BlockOnboarding::VariantTwo),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in BlockOnboarding",
+                s
+            )),
+        }
+    }
+}

app/src/experiments/improved_palette_search_layer.rs

@@ -0,0 +1,70 @@
+use crate::experiments::{BucketRange, Experiment, Layer};
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use std::str::FromStr;
+use warpui::AppContext;
+
+lazy_static! {
+    pub static ref IMPROVED_PALETTE_SEARCH_LAYER: Layer = Layer {
+        name: "ImprovedPaletteSearchLayer",
+        hasher_seeds: (20250401, 1917),
+        traffic_allocations: HashMap::from([
+            (ImprovedPaletteSearch::Control.get_group_id(), 0.0),
+            (ImprovedPaletteSearch::Experiment.get_group_id(), 100.0),
+        ]),
+        bucket_ranges: vec![
+            BucketRange::new(ImprovedPaletteSearch::Control, 0..0),
+            BucketRange::new(ImprovedPaletteSearch::Experiment, 0..1000),
+        ]
+    };
+}
+
+/// An experiment to test the difference between the original search and improved full text search
+#[derive(Debug)]
+pub enum ImprovedPaletteSearch {
+    /// Old fuzzy search
+    Control,
+    /// Tentivy full-text search
+    Experiment,
+}
+
+const IMPROVED_PALETTE_SEARCH_CONTROL: &str = "Control";
+const IMPROVED_PALETTE_SEARCH_EXPERIMENT: &str = "ImprovedPaletteSearchExperiment";
+
+impl Experiment<ImprovedPaletteSearch> for ImprovedPaletteSearch {
+    fn name() -> &'static str {
+        "ImprovedPaletteSearch"
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            Self::Control => IMPROVED_PALETTE_SEARCH_CONTROL,
+            Self::Experiment => IMPROVED_PALETTE_SEARCH_EXPERIMENT,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        false
+    }
+}
+
+impl FromStr for ImprovedPaletteSearch {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            IMPROVED_PALETTE_SEARCH_CONTROL => Ok(Self::Control),
+            IMPROVED_PALETTE_SEARCH_EXPERIMENT => Ok(Self::Experiment),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in ImprovedPaletteSearch",
+                s
+            )),
+        }
+    }
+}
+
+impl ImprovedPaletteSearch {
+    pub fn improved_search_enabled(ctx: &mut AppContext) -> bool {
+        matches!(Self::get_group(ctx), Some(Self::Experiment))
+    }
+}

app/src/experiments/login_layer.rs

@@ -0,0 +1,65 @@
+use super::{BucketRange, Experiment, Layer};
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use std::str::FromStr;
+
+lazy_static! {
+    pub static ref LOGIN_LAYER: Layer = Layer {
+        name: "LoginLayer",
+        hasher_seeds: (100, 9000),
+        traffic_allocations: HashMap::from([
+            (AuthFlowInstructions::Control.get_group_id(), 25.0),
+            (AuthFlowInstructions::Experiment.get_group_id(), 25.0),
+        ]),
+        bucket_ranges: vec![
+            BucketRange::new(AuthFlowInstructions::Control, 0..250),
+            BucketRange::new(AuthFlowInstructions::Experiment, 250..500),
+        ]
+    };
+}
+
+const AUTH_FLOW_INSTRUCTIONS_CONTROL: &str = "Control";
+const AUTH_FLOW_INSTRUCTIONS_EXPERIMENT: &str = "AuthFlowInstructionsExperiment";
+
+/// An experiment to test retention when explicitly instructing users to go to
+/// their browser to continue the authentication flow.
+pub enum AuthFlowInstructions {
+    /// The existing auth flow.
+    Control,
+
+    /// An auth flow that explicitly instructs users to go to their browser
+    /// to continue authenticating.
+    Experiment,
+}
+
+impl Experiment<AuthFlowInstructions> for AuthFlowInstructions {
+    fn name() -> &'static str {
+        "AuthFlowInstructions"
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            Self::Control => AUTH_FLOW_INSTRUCTIONS_CONTROL,
+            Self::Experiment => AUTH_FLOW_INSTRUCTIONS_EXPERIMENT,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        false
+    }
+}
+
+impl FromStr for AuthFlowInstructions {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            AUTH_FLOW_INSTRUCTIONS_CONTROL => Ok(Self::Control),
+            AUTH_FLOW_INSTRUCTIONS_EXPERIMENT => Ok(Self::Experiment),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in AuthFlowInstructions",
+                s
+            )),
+        }
+    }
+}

app/src/experiments/mod.rs

@@ -0,0 +1,408 @@
+//! A framework for running A/B tests within Warp.
+//!
+//! Before starting, please read the usage guide on Notion. The guide explains
+//! some important constraints that are required for proper use of the framework
+//! that we are not able to assert through automated testing.
+//! https://www.notion.so/warpdev/Experiment-Framework-Guide-88954c36a0c3469ea57b427b58249d5f?pvs=4
+
+mod block_onboarding_layer;
+mod login_layer;
+mod rendering;
+pub use block_onboarding_layer::{BlockOnboarding, BLOCK_ONBOARDING_LAYER};
+pub use improved_palette_search_layer::{ImprovedPaletteSearch, IMPROVED_PALETTE_SEARCH_LAYER};
+pub use login_layer::{AuthFlowInstructions, LOGIN_LAYER};
+use warp_core::user_preferences::GetUserPreferences as _;
+
+use crate::auth::auth_state::AuthStateProvider;
+use crate::channel::{Channel, ChannelState};
+use anyhow::Result;
+use dashmap::DashMap;
+use lazy_static::lazy_static;
+use std::fmt;
+use std::marker::Copy;
+use std::ops::Range;
+use std::str::FromStr;
+use std::{collections::HashMap, hash::Hasher};
+
+use warpui::{AppContext, SingletonEntity};
+
+use crate::send_telemetry_sync_from_app_ctx;
+
+/// Number of buckets we are using to partition user traffic. The largest valid
+/// bucket index is NUM_BUCKETS - 1.
+const NUM_BUCKETS: u16 = 1000;
+
+const EXPERIMENT_OVERRIDES_KEY: &str = "ExperimentOverrides";
+
+#[allow(dead_code)]
+const INVALID_GROUP_ASSIGNMENT_ERR: &str =
+    "Invalid group assignment, deriving group from experiment id instead";
+#[allow(dead_code)]
+const INVALID_USER_OVERRIDE_ERR: &str =
+    "Invalid user override, deriving group from experiment id instead";
+#[allow(dead_code)]
+const NO_LAYER_FOUND_ERR: &str = "No layer found for the experiment";
+
+lazy_static! {
+    /// In-memory map that caches users' group assignments so we don't have to calculate
+    /// it from their anonymous id each time. Also keeps track of experiment overrides.
+    /// Key is the name of the experiment, and the value is the variant name.
+    // TODO(daniel): Account for user logout. Currently the cached group assignments
+    // and anonymous id persist even on logout, which may not be the correct behavior.
+    static ref GROUP_ASSIGNMENTS: DashMap<&'static str, &'static str> = DashMap::new();
+
+    /// In-memory map that stores the user's local overrides. This map differs from
+    /// GROUP_ASSIGNMENTS as it uses owned strings to store the overrides read from
+    /// user defaults. The data follows the same structure as GROUP_ASSIGNMENTS: the
+    /// keys are experiment names and the values are the variant names.
+    static ref USER_OVERRIDES: DashMap<String, String> = DashMap::new();
+
+    /// All of the layers currently enabled in the application. A layer must be added
+    /// to this vector in order to use any experiments in that layer. Trying to use an
+    /// experiment in a layer not in this vector will panic in local builds, or result
+    /// in users never being assigned to the experiment in non-local builds.
+    ///
+    /// EMPTY_LAYER is not included here, since we will never add experiments to it,
+    /// and so users can never be assigned to experiments in EMPTY_LAYER.
+    static ref LAYERS: Vec<&'static Layer> = vec![
+        &*LOGIN_LAYER,
+        &*BLOCK_ONBOARDING_LAYER,
+        &*rendering::LAYER,
+        &*IMPROVED_PALETTE_SEARCH_LAYER,
+    ];
+
+    /// Mapping of experiments to their respective layers. The mappings are built up
+    /// during bootstrap. The keys are experiment names.
+    static ref EXPERIMENT_LAYER_MAPPINGS: DashMap<&'static str, &'static Layer> = DashMap::new();
+
+    /// A no-op layer. This layer is only used if an error state occurs where there is
+    /// no layer mapping for a given experiment - the empty layer is returned and the
+    /// user will never be assigned to the experiment.
+    static ref EMPTY_LAYER: Layer = Layer {
+        name: "EmptyLayer",
+        hasher_seeds: (1, 1),
+        traffic_allocations: HashMap::new(),
+        bucket_ranges: Vec::new()
+    };
+}
+
+/// A range of buckets associated with an experiment group.
+#[derive(Clone)]
+struct BucketRange {
+    /// The group to assign this range of buckets to.
+    group: GroupId,
+    /// The range of buckets.
+    range: Range<u16>,
+}
+
+impl BucketRange {
+    // Ignoring the warning that appears when there are no experiments running currently.
+    #[allow(dead_code)]
+    fn new<T>(exp: impl Experiment<T>, range: Range<u16>) -> Self
+    where
+        T: Experiment<T>,
+    {
+        Self {
+            group: exp.get_group_id(),
+            range,
+        }
+    }
+}
+
+/// A unique id representing an experiment group. Contains the name of the
+/// experiment and the variant that it represents.
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub struct GroupId {
+    experiment: &'static str,
+    variant: &'static str,
+}
+
+impl fmt::Display for GroupId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}::{}", self.experiment, self.variant)
+    }
+}
+
+/// A data structure used to define user traffic allocations to experiment groups.
+/// Each layer divides users into buckets 0-999, each covering 0.1% of users. Ranges
+/// of buckets are allocated to experiment groups to route users to experiments.
+///
+/// All groups of an experiment must live in a single layer. All experiments in a layer
+/// will be mutually exclusive, but this is not true for experiments located in different
+/// layers.
+///
+/// Newly created layers should be added to the `LAYERS` vector to be picked up by
+/// automated tests that provide a basic guarantee of correctness.
+///
+/// For more info, see the tech doc: https://docs.google.com/document/d/1BEEeT1Ia7bK-ExK9w-FJKNkEZaRNwTATqZAmX55VldY/edit?usp=sharing
+#[allow(dead_code)]
+pub struct Layer {
+    /// Name of the layer.
+    name: &'static str,
+    /// Seeds used to construct the hasher for this layer.
+    hasher_seeds: (u64, u64),
+    /// The amount of traffic allocated to each experiment group in this layer,
+    /// should be written as percentages: e.g. 20.5 is 20.5%. The values here
+    /// are a safeguard against user error in specifying the bucket ranges,
+    /// which is the sole authority over assigning users to experiments.
+    traffic_allocations: HashMap<GroupId, f64>,
+    /// Ranges of buckets allocated to experiment groups. When increasing the
+    /// allocations for an experiment, we want to ensure users that were previously
+    /// in the experiment remain in the same group. This involves fragmenting the
+    /// bucket ranges for each group. See the `Increasing experiment traffic
+    /// allocations` section in the Notion guide for more details.
+    bucket_ranges: Vec<BucketRange>,
+}
+
+#[allow(dead_code)]
+impl Layer {
+    /// Returns the name of the layer.
+    fn name(&self) -> &'static str {
+        self.name
+    }
+
+    /// Returns the seeds used to construct the hasher for this layer. These
+    /// will stay consistent across app runs and will be unique for each layer.
+    fn hasher_seeds(&self) -> (u64, u64) {
+        self.hasher_seeds
+    }
+
+    /// Searches for a bucket range that includes the given bucket. If found, returns
+    /// the experiment group for that range, or None if no satisfying range was found.
+    fn get_group_for_bucket(&self, bucket: u16) -> Option<GroupId> {
+        if bucket >= NUM_BUCKETS {
+            log::error!("User assigned a bucket greater than the max: {bucket}");
+            return None;
+        }
+        for BucketRange { group, range } in self.bucket_ranges.iter() {
+            if range.contains(&bucket) {
+                return Some(*group);
+            }
+        }
+        None
+    }
+
+    /// Determines the assigned bucket based on a hash of the anonymous id. The
+    /// returned bucket will be in the range 0-999 (inclusive) and is deterministic.
+    fn assigned_bucket(&self, anonymous_id: &str) -> u16 {
+        let (seed_1, seed_2) = self.hasher_seeds();
+        let hash = {
+            let mut hasher = siphasher::sip::SipHasher::new_with_keys(seed_1, seed_2);
+            hasher.write(anonymous_id.as_bytes());
+            hasher.finish()
+        };
+        (hash % 1000) as u16
+    }
+
+    /// Returns the experiment group that the anonymous id is assigned to in this
+    /// layer, if it exists. Returns None otherwise.
+    fn get_assigned_group<T>(&self, anonymous_id: &str) -> Option<T>
+    where
+        T: Experiment<T>,
+        <T as FromStr>::Err: fmt::Debug,
+    {
+        let bucket = self.assigned_bucket(anonymous_id);
+        let group_id = self.get_group_for_bucket(bucket)?;
+
+        // We ignore errors converting group id to T as users can be assigned to a group
+        // in another experiment in this layer, in which case the conversion would fail.
+        T::from_group_id(group_id).ok()
+    }
+}
+
+/// Trait to be implemented by all experiments created for A/B testing, with T
+/// being the type of the experiment itself.
+pub trait Experiment<T: Experiment<T>>: FromStr {
+    /// Returns the name associated with this experiment.
+    fn name() -> &'static str;
+
+    /// Returns the layer this experiment is in. Returns the empty layer if an error
+    /// occurs and no experiment-layer mapping exists for this layer.
+    fn layer() -> &'static Layer {
+        match EXPERIMENT_LAYER_MAPPINGS.get(Self::name()) {
+            Some(layer) => *layer,
+            None => {
+                if cfg!(debug_assertions) {
+                    panic!("{}: {}", NO_LAYER_FOUND_ERR, Self::name());
+                } else {
+                    log::error!("{}: {}", NO_LAYER_FOUND_ERR, Self::name());
+                }
+                &EMPTY_LAYER
+            }
+        }
+    }
+
+    /// Returns the string representation of the current experiment variant.
+    fn variant(&self) -> &'static str;
+
+    /// Whether or not we allow end users to manually override their assigned
+    /// group by modifying their user preferences.
+    ///
+    /// This should only return true if the "experiment" isn't actually an A/B
+    /// test with a hypothesis (e.g. the experiment is used for an incremental
+    /// rollout). Otherwise, the randomness of the data should be preserved and
+    /// we should not allow overrides.
+    ///
+    /// This method should generally not be used in isolation. Instead, use
+    /// [`Self::can_use_user_override`], which calls this method _and_ checks
+    /// the current channel to see if overrides are allowed.
+    fn allow_user_overrides_in_stable() -> bool;
+
+    /// Returns the group id from the current experiment group.
+    fn get_group_id(&self) -> GroupId {
+        GroupId {
+            experiment: Self::name(),
+            variant: self.variant(),
+        }
+    }
+
+    /// Parses a group id to return the associated experiment. Will fail if the
+    /// group id is ill-formatted or is not an arm of this experiment.
+    fn from_group_id(group_id: GroupId) -> Result<T>
+    where
+        <T as FromStr>::Err: fmt::Debug,
+    {
+        if group_id.experiment != Self::name() {
+            return Err(anyhow::anyhow!(
+                "Cannot parse a GroupId of experiment {} into experiment {}",
+                group_id.experiment,
+                Self::name()
+            ));
+        }
+        T::from_str(group_id.variant)
+            .map_err(|e| anyhow::anyhow!("Failed to parse GroupId: {:?}", e))
+    }
+
+    /// Gets the assigned group of the experiment for the current user. Returns None
+    /// if the user is not in this experiment.
+    ///
+    /// TODO: we should investigate if we can suffice with just a AppContext
+    /// here to allow `get_group` to be used when a AppContext isn't available
+    /// (e.g. when rendering a view). We currently need it because `get_group`
+    /// might emit telemetry.
+    fn get_group(ctx: &mut AppContext) -> Option<T>
+    where
+        <T as FromStr>::Err: fmt::Debug,
+    {
+        // Check if we have cached the group assignment in memory.
+        if let Some(variant) = GROUP_ASSIGNMENTS.get(Self::name()) {
+            match T::from_str(*variant) {
+                Ok(group) => return Some(group),
+                Err(e) => {
+                    if cfg!(debug_assertions) {
+                        panic!("{INVALID_GROUP_ASSIGNMENT_ERR}: {e:?}");
+                    } else {
+                        log::error!("{INVALID_GROUP_ASSIGNMENT_ERR}: {e:?}");
+                    }
+                }
+            };
+        }
+
+        let mut assigned_group = None;
+
+        // Check for user override. Only used in local and dev builds or if the
+        // this experiment allows overrides.
+        if Self::can_use_user_override(ChannelState::channel()) {
+            if let Some(variant) = USER_OVERRIDES.get(Self::name()) {
+                match T::from_str(&variant) {
+                    Ok(group) => assigned_group = Some(group),
+                    Err(e) => {
+                        log::error!("{INVALID_USER_OVERRIDE_ERR}: {e:?}");
+                    }
+                };
+            }
+        }
+
+        // If there was no override, derive the assignment from the user's anonymous id.
+        if assigned_group.is_none() {
+            let anonymous_id = AuthStateProvider::as_ref(ctx).get().anonymous_id();
+            assigned_group = Self::layer().get_assigned_group(&anonymous_id);
+
+            if let Some(group) = assigned_group.as_ref() {
+                let group_assignment = group.variant();
+                // Send synchronously since this we rely on this event to collect experiment data.
+                send_telemetry_sync_from_app_ctx!(
+                    crate::server::telemetry::TelemetryEvent::ExperimentTriggered {
+                        experiment: Self::name(),
+                        layer: Self::layer().name(),
+                        group_assignment,
+                    },
+                    ctx
+                );
+            }
+        }
+
+        // If the user is in a group for this experiment, cache the result of
+        // the work above and do any one-time accounting.
+        if let Some(group) = assigned_group.as_ref() {
+            GROUP_ASSIGNMENTS.insert(Self::name(), group.variant());
+
+            #[cfg(feature = "crash_reporting")]
+            {
+                let tag_name = format!("warp.experiments.{}", Self::name());
+                crate::crash_reporting::set_tag(&tag_name, group.variant());
+            }
+        }
+
+        assigned_group
+    }
+
+    /// Overrides the user's assigned group for this experiment for the remainder
+    /// of the app lifecycle.
+    #[allow(dead_code)]
+    fn set_override<S: Experiment<T>>(override_group: S) {
+        GROUP_ASSIGNMENTS.insert(Self::name(), override_group.variant());
+    }
+
+    /// Returns whether user overrides should be allowed based on the channel and
+    /// experiment setting. User overrides are always allowed in Local and Dev
+    /// channels, or if the specific experiment supports overrides.
+    fn can_use_user_override(channel: Channel) -> bool {
+        Self::allow_user_overrides_in_stable() || channel.is_dogfood()
+    }
+}
+
+/// Creates the experiment-layer mappings given a list of layers. This method assumes
+/// that each experiment is included in a single layer, which should be asserted by
+/// the validation tests.
+fn create_experiment_layer_mappings(layers: &[&'static Layer]) {
+    for layer in layers.iter() {
+        for group_id in layer.traffic_allocations.keys() {
+            EXPERIMENT_LAYER_MAPPINGS.insert(group_id.experiment, layer);
+        }
+    }
+}
+
+/// Reads in the user overrides. Overrides should be a comma delimited list of
+/// group ids under the EXPERIMENT_OVERRIDES_KEY. For example:
+/// "ExperimentOverrides": "Experiment1::GroupA,Experiment2::GroupB"
+///
+/// Note that an override will only be applied if the current channel or the
+/// specific experiment supports overrides.
+fn set_user_overrides(ctx: &mut AppContext) {
+    if let Some(overrides) = ctx
+        .private_user_preferences()
+        .read_value(EXPERIMENT_OVERRIDES_KEY)
+        .unwrap_or_default()
+    {
+        for group in overrides.split(',') {
+            if let Some((experiment_name, variant)) = group.split_once("::") {
+                USER_OVERRIDES.insert(experiment_name.into(), variant.into());
+            }
+        }
+    }
+}
+
+/// Initializes the experiment framework.
+pub fn init(ctx: &mut AppContext) {
+    create_experiment_layer_mappings(&LAYERS);
+    set_user_overrides(ctx);
+}
+
+#[cfg(test)]
+#[path = "mod_tests.rs"]
+mod tests;
+
+mod improved_palette_search_layer;
+#[cfg(test)]
+mod validation_tests;

app/src/experiments/mod_tests.rs

@@ -0,0 +1,182 @@
+use super::*;
+
+lazy_static! {
+    pub static ref TEST_LAYER: Layer = Layer {
+        name: "TestLayer",
+        hasher_seeds: (1, 2),
+        traffic_allocations: HashMap::from([
+            (TestExperiment::Control.get_group_id(), 10.0,),
+            (TestExperiment::Experiment.get_group_id(), 10.0,),
+            (FooExperiment::Control.get_group_id(), 5.0,),
+            (FooExperiment::Experiment.get_group_id(), 5.0,),
+        ]),
+        bucket_ranges: vec![
+            BucketRange::new(TestExperiment::Control, 0..100),
+            BucketRange::new(TestExperiment::Experiment, 300..400),
+            BucketRange::new(FooExperiment::Control, 500..550),
+            BucketRange::new(FooExperiment::Experiment, 550..600),
+        ],
+    };
+}
+
+const TEST_EXPERIMENT_CONTROL: &str = "Control";
+const TEST_EXPERIMENT_EXPERIMENT: &str = "Experiment";
+
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub enum TestExperiment {
+    Control,
+    Experiment,
+}
+
+impl Experiment<TestExperiment> for TestExperiment {
+    fn name() -> &'static str {
+        "TestExperiment"
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            Self::Control => TEST_EXPERIMENT_CONTROL,
+            Self::Experiment => TEST_EXPERIMENT_EXPERIMENT,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        false
+    }
+}
+
+impl FromStr for TestExperiment {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            TEST_EXPERIMENT_CONTROL => Ok(Self::Control),
+            TEST_EXPERIMENT_EXPERIMENT => Ok(Self::Experiment),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in TestExperiment",
+                s
+            )),
+        }
+    }
+}
+
+const FOO_EXPERIMENT_CONTROL: &str = "Control";
+const FOO_EXPERIMENT_EXPERIMENT: &str = "Experiment";
+
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub enum FooExperiment {
+    Control,
+    Experiment,
+}
+
+impl Experiment<FooExperiment> for FooExperiment {
+    fn name() -> &'static str {
+        "FooExperiment"
+    }
+
+    fn layer() -> &'static Layer {
+        &TEST_LAYER
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            Self::Control => FOO_EXPERIMENT_CONTROL,
+            Self::Experiment => FOO_EXPERIMENT_EXPERIMENT,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        true
+    }
+}
+
+impl FromStr for FooExperiment {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            FOO_EXPERIMENT_CONTROL => Ok(Self::Control),
+            FOO_EXPERIMENT_EXPERIMENT => Ok(Self::Experiment),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in FooExperiment",
+                s
+            )),
+        }
+    }
+}
+
+#[test]
+fn test_get_group_for_bucket_finds_correct_range() {
+    let bucket_range = TEST_LAYER.bucket_ranges.get(1).unwrap();
+    let expected = Some(bucket_range.group);
+    let lower_bound = bucket_range.range.start;
+    let upper_bound_inclusive = bucket_range.range.end - 1;
+
+    let res = TEST_LAYER.get_group_for_bucket(lower_bound);
+    assert_eq!(res, expected);
+    let res = TEST_LAYER.get_group_for_bucket((lower_bound + upper_bound_inclusive) / 2);
+    assert_eq!(res, expected);
+    let res = TEST_LAYER.get_group_for_bucket(upper_bound_inclusive);
+    assert_eq!(res, expected);
+}
+
+#[test]
+fn test_get_group_for_bucket_not_in_range() {
+    let expected = None;
+    let bucket_range = TEST_LAYER.bucket_ranges.get(1).unwrap();
+    let lower_bound = bucket_range.range.start;
+    let upper_bound = bucket_range.range.end;
+
+    let res = TEST_LAYER.get_group_for_bucket(lower_bound - 1);
+    assert_eq!(res, expected);
+    let res = TEST_LAYER.get_group_for_bucket(upper_bound);
+    assert_eq!(res, expected);
+}
+
+#[test]
+fn test_from_group_id_errors_if_incorrect_experiment() {
+    let group_id = GroupId {
+        experiment: FooExperiment::name(),
+        variant: FOO_EXPERIMENT_CONTROL,
+    };
+    let res = TestExperiment::from_group_id(group_id);
+
+    assert!(res.is_err())
+}
+
+#[test]
+fn test_create_experiment_layer_mappings() {
+    let layers = vec![&*TEST_LAYER];
+    create_experiment_layer_mappings(&layers);
+
+    assert_eq!(EXPERIMENT_LAYER_MAPPINGS.len(), 2);
+    assert_eq!(
+        EXPERIMENT_LAYER_MAPPINGS
+            .get(TestExperiment::name())
+            .expect("Layer mapping should have been created for TestExperiment.")
+            .name(),
+        TEST_LAYER.name()
+    );
+    assert_eq!(
+        EXPERIMENT_LAYER_MAPPINGS
+            .get(FooExperiment::name())
+            .expect("Layer mapping should have been created for FooExperiment.")
+            .name(),
+        TEST_LAYER.name()
+    );
+}
+
+#[test]
+fn test_allow_user_overrides() {
+    // Case 1: Experiment and channel don't allow overrides.
+    assert!(!TestExperiment::can_use_user_override(Channel::Stable));
+
+    // Case 2: Experiment doesn't allow overrides, channel does.
+    assert!(TestExperiment::can_use_user_override(Channel::Dev));
+
+    // Case 3: Experiment and channel allow overrides.
+    assert!(FooExperiment::can_use_user_override(Channel::Dev));
+
+    // Case 4: Experiment allows overrides, channel doesn't.
+    assert!(FooExperiment::can_use_user_override(Channel::Stable));
+}

app/src/experiments/rendering.rs

@@ -0,0 +1,14 @@
+use std::collections::HashMap;
+
+use lazy_static::lazy_static;
+
+use super::Layer;
+
+lazy_static! {
+    pub(super) static ref LAYER: Layer = Layer {
+        name: "RenderingLayer",
+        hasher_seeds: (20241104, 132105),
+        traffic_allocations: HashMap::from([]),
+        bucket_ranges: vec![]
+    };
+}

app/src/experiments/validation_tests.rs

@@ -0,0 +1,110 @@
+use super::*;
+use std::cmp::max;
+use std::collections::HashSet;
+
+#[test]
+fn test_all_layers_have_unique_seeds() {
+    let mut prev_seen_seeds = HashSet::new();
+    for layer in LAYERS.iter() {
+        let layer_seeds = layer.hasher_seeds();
+        assert!(
+            !prev_seen_seeds.contains(&layer_seeds),
+            "There are two layers that use the seeds {layer_seeds:?} to construct a hasher"
+        );
+        prev_seen_seeds.insert(layer.hasher_seeds());
+    }
+}
+
+#[test]
+fn test_valid_bucket_ranges() {
+    for layer in LAYERS.iter() {
+        for bucket_range in &layer.bucket_ranges {
+            assert!(
+                bucket_range.range.start <= bucket_range.range.end,
+                "The start range should be <= the end range for group {:?} in layer {}",
+                bucket_range.group,
+                layer.name()
+            );
+            assert!(
+                bucket_range.range.start <= NUM_BUCKETS,
+                "{} is not a valid range start for group {:?} in layer {}",
+                bucket_range.range.start,
+                bucket_range.group,
+                layer.name()
+            );
+            assert!(
+                bucket_range.range.end <= NUM_BUCKETS,
+                "{} is not a valid range end for group {:?} in layer {}",
+                bucket_range.range.end,
+                bucket_range.group,
+                layer.name()
+            );
+        }
+    }
+}
+
+#[test]
+fn test_no_overlapping_bucket_ranges() {
+    for layer in LAYERS.iter() {
+        // Sort by ranges by range start.
+        let mut sorted_ranges = layer.bucket_ranges.to_vec();
+        sorted_ranges.sort_by_key(|r| r.range.start);
+
+        // Track the largest range end we have seen so far.
+        let mut prev_end: Option<u16> = None;
+        for bucket_range in sorted_ranges {
+            if let Some(prev_end_range) = prev_end {
+                // If the current range start < a previous end range, they must overlap
+                // since the current range start >= the previous range start.
+                assert!(
+                    bucket_range.range.start >= prev_end_range,
+                    "Overlapping bucket ranges in layer: {}",
+                    layer.name()
+                );
+                prev_end = Some(max(bucket_range.range.end, prev_end_range));
+            } else {
+                prev_end = Some(bucket_range.range.end);
+            }
+        }
+    }
+}
+
+#[test]
+fn test_bucket_ranges_sum_to_traffic_allocations() {
+    for layer in LAYERS.iter() {
+        let mut allocations: HashMap<GroupId, f64> = HashMap::new();
+        // Manually sum bucket range sizes to see if they match the layer's defined
+        // traffic ranges
+        for bucket_range in &layer.bucket_ranges {
+            let range_size = bucket_range.range.end - bucket_range.range.start;
+            let traffic_percentage = (range_size as f64) / 10.0;
+
+            let allocation = allocations.entry(bucket_range.group).or_insert(0.0);
+            *allocation += traffic_percentage;
+        }
+        assert!(
+            allocations == layer.traffic_allocations,
+            "Bucket ranges do not sum up to the traffic ranges for layer {}",
+            layer.name()
+        );
+    }
+}
+
+#[test]
+fn test_no_experiments_in_multiple_layers() {
+    let mut prev_seen_experiments: HashSet<String> = HashSet::new();
+    for layer in LAYERS.iter() {
+        // Set of experiments in the current layer
+        let mut cur_experiments: HashSet<String> = HashSet::new();
+        for group_id in layer.traffic_allocations.keys() {
+            // group_id will follow format "{experiment_name}::{variant_name}"
+            let experiment_name = group_id.to_string().split_once(':').unwrap().0.to_owned();
+            assert!(
+                !prev_seen_experiments.contains(&experiment_name),
+                "Experiment {experiment_name} is used in multiple layers"
+            );
+            cur_experiments.insert(experiment_name);
+        }
+        prev_seen_experiments.extend(cur_experiments);
+    }
+}