first pass of merging in warp (doesn't build)

crates/asset_cache/Cargo.toml

@@ -10,6 +10,7 @@ license.workspace = true
 anyhow.workspace = true
 async-compat.workspace = true
 async-fs.workspace = true
+base64.workspace = true
 bytes.workspace = true
 cfg-if.workspace = true
 futures.workspace = true

crates/asset_cache/src/lib.rs

@@ -3,9 +3,9 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use anyhow::Result;
-use async_fs::{OpenOptions, create_dir_all};
+use base64::Engine as _;
+use base64::prelude::BASE64_STANDARD;
 use bytes::Bytes;
-use futures::AsyncWriteExt;
 use galaxyui_core::assets::asset_cache::{
     Asset, AssetCache, AssetSource, AssetState, AsyncAssetId, AsyncAssetType,
 };
@@ -15,6 +15,12 @@ use reqwest::Url;
 pub struct UrlAssetWithoutPersistence;
 impl AsyncAssetType for UrlAssetWithoutPersistence {}
 
+/// Namespace marker for inline base64 `data:` URI async asset sources.
+pub struct DataUriAsset;
+impl AsyncAssetType for DataUriAsset {}
+
+pub const MAX_DATA_URI_PAYLOAD_BYTES: usize = 16 * 1024 * 1024;
+
 /// Namespace marker for URL-based async asset sources with persistence.
 ///
 /// This is intentionally separate from `UrlAssetWithoutPersistence` to allow
@@ -40,6 +46,63 @@ pub fn url_source(url: impl Into<String>) -> AssetSource {
     }
 }
 
+/// Returns `true` if `source` is a base64 `data:` URI whose encoded payload
+/// exceeds `MAX_DATA_URI_PAYLOAD_BYTES`. Non-`data:` URIs and `data:` URIs
+/// without a `;base64` marker return `false`.
+pub fn data_uri_exceeds_limit(source: &str) -> bool {
+    let Some((header, payload)) = source
+        .strip_prefix("data:")
+        .and_then(|rest| rest.split_once(','))
+    else {
+        return false;
+    };
+    header
+        .split(';')
+        .any(|segment| segment.eq_ignore_ascii_case("base64"))
+        && payload.len() > MAX_DATA_URI_PAYLOAD_BYTES
+}
+
+/// Creates an [`AssetSource::Async`] that decodes an inline base64 `data:` URI
+/// (e.g. `data:image/png;base64,<payload>`) into its raw bytes.
+pub fn data_uri_source(source: &str) -> Option<AssetSource> {
+    // data:[<mediatype>][;base64],<payload>
+    let (header, payload) = source.strip_prefix("data:")?.split_once(',')?;
+    if !header
+        .split(';')
+        .any(|segment| segment.eq_ignore_ascii_case("base64"))
+    {
+        return None;
+    }
+
+    // `source` is untrusted; reject oversized payloads before cloning/decoding
+    if data_uri_exceeds_limit(source) {
+        return None;
+    }
+
+    // Derive a compact, stable cache key from the full URI so identical payloads
+    // dedupe and we don't retain the (potentially large) data URI as the key.
+    let mut hasher = DefaultHasher::new();
+    source.hash(&mut hasher);
+    let id = format!("{:x}", hasher.finish());
+
+    // base64 payloads may contain embedded whitespace/newlines; strip it before
+    // decoding.
+    let payload: String = payload.chars().filter(|c| !c.is_whitespace()).collect();
+
+    Some(AssetSource::Async {
+        id: AsyncAssetId::new::<DataUriAsset>(id),
+        fetch: Arc::new(move || {
+            let payload = payload.clone();
+            Box::pin(async move {
+                BASE64_STANDARD
+                    .decode(payload.as_bytes())
+                    .map(Bytes::from)
+                    .map_err(Into::into)
+            })
+        }),
+    })
+}
+
 /// Creates an [`AssetSource::Async`] that fetches bytes from the given URL,
 /// persisting them to a file under `cache_dir` for future reads.
 pub fn url_source_with_persistence(url: impl Into<String>, cache_dir: &Path) -> AssetSource {
@@ -89,7 +152,7 @@ async fn fetch_file_to_memory(url: Url) -> Result<Bytes, anyhow::Error> {
             let response = async_compat::Compat::new(async move { reqwest::get(url).await }).await?;
         }
     }
-    let content = response.bytes().await?;
+    let content = response.error_for_status()?.bytes().await?;
     Ok(content)
 }
 
@@ -109,7 +172,11 @@ fn get_file_path_for_asset(url: &Url, cache_dir: &Path) -> PathBuf {
     cache_dir.join(filename)
 }
 
+#[cfg(not(target_family = "wasm"))]
 async fn persist_bytes(bytes: &Bytes, file: &Path) {
+    use async_fs::{OpenOptions, create_dir_all};
+    use futures::AsyncWriteExt;
+
     let Some(parent_folder) = file.parent() else {
         log::error!("attempted to write cache file in filesystem root");
         return;
@@ -142,6 +209,11 @@ async fn persist_bytes(bytes: &Bytes, file: &Path) {
     };
 }
 
+#[cfg(target_family = "wasm")]
+async fn persist_bytes(_bytes: &Bytes, file: &Path) {
+    log::debug!("Cannot persist asset to {} on the web", file.display());
+}
+
 async fn fetch_file_and_persist_bytes(url: Url, file: Option<PathBuf>) -> Result<Bytes> {
     let result = fetch_file_to_memory(url).await;
 
@@ -174,3 +246,7 @@ async fn fetch_asset_from_url(url: Url, file: Option<PathBuf>) -> Result<Bytes>
         _ => fetch_file_and_persist_bytes(url, file).await,
     }
 }
+
+#[cfg(test)]
+#[path = "lib_tests.rs"]
+mod tests;

crates/asset_cache/src/lib_tests.rs

@@ -0,0 +1,78 @@
+use super::*;
+
+/// Drive the fetch closure of an [`AssetSource::Async`] to completion.
+fn fetch_bytes(source: &AssetSource) -> Result<Bytes> {
+    match source {
+        AssetSource::Async { fetch, .. } => futures::executor::block_on(fetch()),
+        other => panic!("expected an Async source, got {other:?}"),
+    }
+}
+
+#[test]
+fn data_uri_source_decodes_base64_payload() {
+    let source = data_uri_source("data:image/png;base64,iVBORw0KGgo=")
+        .expect("base64 data URI should produce a source");
+    let bytes = fetch_bytes(&source).expect("payload should decode");
+    assert_eq!(
+        bytes.as_ref(),
+        BASE64_STANDARD.decode("iVBORw0KGgo=").unwrap().as_slice()
+    );
+}
+
+#[test]
+fn data_uri_source_strips_embedded_whitespace() {
+    // base64 payloads saved in notebooks frequently contain newlines.
+    let source =
+        data_uri_source("data:image/png;base64,iVBO\nRw0K Ggo=").expect("should produce a source");
+    let bytes = fetch_bytes(&source).expect("payload should decode after stripping whitespace");
+    assert_eq!(
+        bytes.as_ref(),
+        BASE64_STANDARD.decode("iVBORw0KGgo=").unwrap().as_slice()
+    );
+}
+
+#[test]
+fn data_uri_source_rejects_non_base64_data_uris() {
+    assert!(data_uri_source("https://example.com/a.png").is_none());
+    assert!(data_uri_source("/abs/path.png").is_none());
+    assert!(data_uri_source("relative/path.png").is_none());
+    // A `data:` URI without the `;base64` marker is not a renderable asset.
+    assert!(data_uri_source("data:text/plain,hello").is_none());
+    // A `data:` URI without a comma separator is malformed.
+    assert!(data_uri_source("data:image/png;base64").is_none());
+}
+
+#[test]
+fn data_uri_source_invalid_base64_fails_on_fetch() {
+    // Detection succeeds on the prefix/marker, but decoding the bad payload
+    // surfaces as a fetch error (FailedToLoad) rather than a panic.
+    let source =
+        data_uri_source("data:image/png;base64,not valid base64!").expect("detected as data URI");
+    assert!(fetch_bytes(&source).is_err());
+}
+
+#[test]
+fn data_uri_source_rejects_oversized_payload() {
+    // An untrusted, oversized payload must be rejected before it is cloned or
+    // decoded, so it never produces an asset source.
+    let huge = "A".repeat(MAX_DATA_URI_PAYLOAD_BYTES + 1);
+    let source = format!("data:image/png;base64,{huge}");
+    assert!(data_uri_source(&source).is_none());
+}
+
+#[test]
+fn data_uri_exceeds_limit_flags_only_oversized_base64_payloads() {
+    let huge = "A".repeat(MAX_DATA_URI_PAYLOAD_BYTES + 1);
+    assert!(data_uri_exceeds_limit(&format!(
+        "data:image/png;base64,{huge}"
+    )));
+
+    // In-limit payloads, non-base64 `data:` URIs, and non-`data:` sources are
+    // not flagged as oversized.
+    assert!(!data_uri_exceeds_limit(
+        "data:image/png;base64,iVBORw0KGgo="
+    ));
+    assert!(!data_uri_exceeds_limit(&format!("data:text/plain,{huge}")));
+    assert!(!data_uri_exceeds_limit("https://example.com/a.png"));
+    assert!(!data_uri_exceeds_limit("relative/path.png"));
+}