public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

first pass of merging in warp (doesn't build)

Browse Source
This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/markdown_parser/src/html_parser.rs
+12 -13
View File
@@ -1,21 +1,20 @@
use std::cell::Cell;
use std::collections::{HashMap, VecDeque};
use std::rc::Rc;
use anyhow::Result;
use serde_yaml::{Mapping, Value};
use std::{
cell::Cell,
collections::{HashMap, VecDeque},
rc::Rc,
};
use html5ever::{
Attribute, ParseOpts, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts,
};
use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::TreeBuilderOpts;
use html5ever::{Attribute, ParseOpts, parse_document};
use markup5ever_rcdom::{Node, NodeData, RcDom};
use serde_yaml::{Mapping, Value};
use crate::markdown_parser::RUNNABLE_BLOCK_MARKDOWN_LANG;
use crate::weight::CustomWeight;
use crate::{
CodeBlockText, FormattedIndentTextInline, FormattedTaskList, FormattedText,
FormattedTextFragment, FormattedTextHeader, FormattedTextInline, FormattedTextLine,
FormattedTextStyles, Hyperlink, OrderedFormattedIndentTextInline,
markdown_parser::RUNNABLE_BLOCK_MARKDOWN_LANG, weight::CustomWeight,
};
// Top element element tags we are not parsing for right now.
@@ -558,7 +557,7 @@ fn parse_code_block_and_language(nodes: &[Rc<Node>]) -> (String, Option<String>)
(text, language)
}
// Parse a HMTL style string into its corresponding name -> value hashmap
// Parse a HTML style string into its corresponding name -> value hashmap
// For example "font-style:italic;font-weight:400" will be parsed into
// {"font-style": "italic", "font-weight": "400"}.
fn parse_style_into_dict(style: &str) -> HashMap<&str, &str> {
@@ -576,5 +575,5 @@ fn parse_style_into_dict(style: &str) -> HashMap<&str, &str> {
}
#[cfg(test)]
#[path = "html_parser_test.rs"]
#[path = "html_parser_tests.rs"]
mod tests;
crates/markdown_parser/src/html_parser_test.rs → crates/markdown_parser/src/html_parser_tests.rs
View File
crates/markdown_parser/src/lib.rs
+5 -3
View File
@@ -1,7 +1,9 @@
use std::any::Any;
use std::collections::VecDeque;
use std::fmt;
use std::fmt::Debug;
use std::ops::Range;
use std::sync::Arc;
use std::{collections::VecDeque, fmt, fmt::Debug};
pub mod html_parser;
pub mod markdown_parser;
@@ -18,9 +20,9 @@ use weight::CustomWeight;
/// Trait for an "action" that can be dispatched via a hyperlink click handler.
/// This purposefully shadows the `Action` trait from `galaxyui`.
///
/// Since `galaxyui` depends on this crate, we can't depend on the `galaxyui::Action` trait directly.
/// Since `galaxyui` depends on this crate, we can't depend on the `galaxyui_core::Action` trait directly.
/// Instead, we create a new trait with a blanket implementation that implicitly results
/// in any `galaxyui::Action` implementing this `Action`.
/// in any `galaxyui_core::Action` implementing this `Action`.
pub trait Action: Any + Debug + Send + Sync {
fn as_any(&self) -> &dyn Any;
}
crates/markdown_parser/src/markdown_parser.rs
+55 -44
View File
@@ -1,32 +1,28 @@
use anyhow::Result;
use itertools::Itertools;
use nom::{
FindToken, IResult, InputIter, InputLength, Parser, Slice,
branch::alt,
bytes::complete::{
is_a, tag, tag_no_case, take, take_till1, take_until, take_while, take_while_m_n,
take_while1,
},
character::{
complete::{char, one_of, satisfy, space0, space1},
is_digit,
},
combinator::{
all_consuming, consumed, eof, fail, flat_map, map, map_parser, recognize, value, verify,
},
error::{ContextError, ErrorKind, ParseError, context, make_error},
multi::{fold_many_m_n, fold_many1, many_m_n, many0},
sequence::{delimited, pair, preceded, terminated, tuple},
};
use serde_yaml::Value;
use std::cell::RefCell;
use anyhow::Result;
use itertools::Itertools;
use nom::branch::alt;
use nom::bytes::complete::{
is_a, tag, tag_no_case, take, take_till1, take_until, take_while, take_while_m_n, take_while1,
};
use nom::character::complete::{char, one_of, satisfy, space0, space1};
use nom::character::is_digit;
use nom::combinator::{
all_consuming, consumed, eof, fail, flat_map, map, map_parser, recognize, value, verify,
};
use nom::error::{ContextError, ErrorKind, ParseError, context, make_error};
use nom::multi::{fold_many_m_n, fold_many1, many_m_n, many0};
use nom::sequence::{delimited, pair, preceded, terminated, tuple};
use nom::{FindToken, IResult, InputIter, InputLength, Parser, Slice};
use serde_yaml::Value;
use crate::{
CodeBlockText, FormattedImage, FormattedIndentTextInline, FormattedTable, FormattedTaskList,
FormattedText, FormattedTextFragment, FormattedTextHeader, FormattedTextInline,
FormattedTextLine, Hyperlink, OrderedFormattedIndentTextInline, TableAlignment,
CodeBlockText, CustomWeight, FormattedImage, FormattedIndentTextInline, FormattedTable,
FormattedTaskList, FormattedText, FormattedTextFragment, FormattedTextHeader,
FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink,
OrderedFormattedIndentTextInline, TableAlignment,
};
use crate::{CustomWeight, FormattedTextStyles};
const HEADER_TAG_MIN_COUNT: usize = 1;
const HEADER_TAG_MAX_COUNT: usize = 6;
@@ -1003,7 +999,7 @@ fn parse_inline<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
c.is_whitespace() || FORMATTING_DELIMITERS.contains(c) || c == '('
});
if can_autolink {
state.push_closed_node(FormattedTextFragment::hyperlink(url, url));
state.push_closed_node(FormattedTextFragment::hyperlink(url.clone(), url));
} else {
state.push_text(url);
}
@@ -1315,7 +1311,7 @@ fn parse_underline<'a>(state: &mut InlineState, remaining: &'a str) -> &'a str {
///
/// This is approximately equivalent to the CommonMark [process emphasis](https://spec.commonmark.org/0.30/#phase-2-inline-structure)
/// algorithm. However:
/// * It omits `openers_bottom`, which is purely a performace optimization.
/// * It omits `openers_bottom`, which is purely a performance optimization.
/// * It uses a `Vec` rather than a linked list, which changes the structure a bit to work with lifetimes.
/// * It also parses [GFM strikethrough](https://github.github.com/gfm/#strikethrough-extension-).
fn process_emphasis(state: &mut InlineState, stack_bottom: Option<usize>) {
@@ -1476,7 +1472,7 @@ fn process_emphasis(state: &mut InlineState, stack_bottom: Option<usize>) {
/// Helper for [`process_emphasis`] that removes `count` delimiters of `kind` from `node`.
///
/// In debug builds, this panics if `node` is not a run of `kind` delimiters.
fn truncate_delimiters(node: &mut FormattedTextFragment, kind: DelimiterKind, count: u8) {
fn truncate_delimiters(node: &mut FormattedTextFragment, kind: DelimiterKind, count: usize) {
let delimiter = kind.as_str();
if cfg!(debug_assertions) {
let text = &node.text;
@@ -1488,7 +1484,7 @@ fn truncate_delimiters(node: &mut FormattedTextFragment, kind: DelimiterKind, co
}
node.text
.truncate(node.text.len() - count as usize * delimiter.len());
.truncate(node.text.len() - count * delimiter.len());
}
/// Helper to merge adjacent text fragments with the same styling. Such fragments might come from:
@@ -1678,7 +1674,7 @@ fn parse_code_span<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
#[derive(Debug, Clone, PartialEq, Eq)]
enum InlineToken<'a> {
/// A run of `count` delimiter characters of `kind`.
Delimiter { kind: DelimiterKind, count: u8 },
Delimiter { kind: DelimiterKind, count: usize },
/// A run of non-delimiter text.
Text(&'a str),
/// A backslash-escaped character.
@@ -1688,8 +1684,9 @@ enum InlineToken<'a> {
/// An entire code span. Code spans have higher precedence than all other inline constructs,
/// so we parse them into discrete tokens.
CodeSpan(&'a str),
/// An autolink URL.
AutoLink(&'a str),
/// An autolink URL. Owned because backslash escapes are processed (e.g., `\.` → `.`),
/// so the result may differ from the input slice.
AutoLink(String),
/// A closing `]` bracket, which triggers link parsing.
LinkEnd,
/// A closing </u>, which triggers underline parsing.
@@ -1703,9 +1700,9 @@ struct Delimiter {
kind: DelimiterKind,
/// The count of repeated delimiter units. This is modified during parsing as delimiters are
/// consumed.
count: u8,
count: usize,
/// The count at the time the delimiter was parsed.
original_count: u8,
original_count: usize,
/// Whether or not this delimiter is active (only applies to link delimiters).
active: bool,
/// The index of the [`FormattedTextFragment`] corresponding to this delimiter.
@@ -1724,7 +1721,7 @@ impl Delimiter {
fn new(
node_index: usize,
kind: DelimiterKind,
count: u8,
count: usize,
preceding_char: Option<char>,
following_char: Option<char>,
) -> Self {
@@ -1777,7 +1774,7 @@ impl Delimiter {
/// Convert this delimiter to literal text.
fn to_text(&self) -> String {
self.kind.as_str().repeat(self.count as usize)
self.kind.as_str().repeat(self.count)
}
/// Whether or not this delimiter can open for the given closing delimiter.
@@ -1817,7 +1814,7 @@ enum DelimiterKind {
impl DelimiterKind {
/// Whether or not `count` is a valid run length for this delimiter.
fn valid_count(self, count: u8) -> bool {
fn valid_count(self, count: usize) -> bool {
match self {
// Emphasis and strong emphasis may be repeated arbitrarily.
DelimiterKind::Asterisk | DelimiterKind::Underscore => true,
@@ -1850,11 +1847,12 @@ fn parse_url_prefix<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
// - starts with "https://" or "http://" or "www."
// - has at least one alphanumeric char after the prefix
// - does not include trailing formatting characters (*, _, ~)
// - backslash escapes are processed (e.g., `\.` → `.`)
fn parse_url<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
i: &'a str,
) -> IResult<&'a str, &'a str, E> {
) -> IResult<&'a str, String, E> {
// TODO: Look into other autolink rules here: https://github.github.com/gfm/#autolinks-extension-
let (_, url) = recognize(tuple((
let (_, raw_url) = recognize(tuple((
parse_url_prefix,
take_till1(|c: char| c.is_whitespace() || "[]<".find_token(c)),
)))(i)?;
@@ -1862,12 +1860,12 @@ fn parse_url<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
// Strip trailing formatting characters (*, _, ~) from the URL.
// Per GFM spec, autolinks should not include trailing punctuation that could be
// markdown formatting delimiters.
let trimmed_len = url
let trimmed_len = raw_url
.trim_end_matches(|c| FORMATTING_DELIMITERS.contains(c))
.len();
// If we trimmed everything after the prefix, the URL is invalid
let min_valid_len = match url.find("://") {
let min_valid_len = match raw_url.find("://") {
Some(pos) => pos + "://".len(),
None => "www.".len(),
};
@@ -1875,9 +1873,22 @@ fn parse_url<'a, E: ContextError<&'a str> + ParseError<&'a str>>(
return Err(nom::Err::Error(make_error(i, ErrorKind::TakeWhile1)));
}
// Return the trimmed URL and adjust remaining
let trimmed_url = &i[..trimmed_len];
let trimmed_raw_url = &i[..trimmed_len];
let new_remaining = &i[trimmed_len..];
// Process backslash escapes within the URL using parse_escape (e.g., `\.` → `.`).
let mut trimmed_url = String::with_capacity(trimmed_len);
let mut remaining = trimmed_raw_url;
while !remaining.is_empty() {
if let Ok((rest, ch)) = parse_escape::<nom::error::Error<&str>>(remaining) {
trimmed_url.push(ch);
remaining = rest;
} else if let Some(ch) = remaining.chars().next() {
trimmed_url.push(ch);
remaining = &remaining[ch.len_utf8()..];
}
}
Ok((new_remaining, trimmed_url))
}
@@ -1957,5 +1968,5 @@ fn parse_block_leading_spaces<'a, E: ContextError<&'a str> + ParseError<&'a str>
}
#[cfg(test)]
#[path = "markdown_parser_test.rs"]
#[path = "markdown_parser_tests.rs"]
mod tests;
crates/markdown_parser/src/markdown_parser_test.rs → crates/markdown_parser/src/markdown_parser_tests.rs
+102 -6
View File
@@ -1,15 +1,12 @@
use nom::{
Finish,
error::{VerboseError, convert_error},
};
use nom::Finish;
use nom::error::{VerboseError, convert_error};
use serde_yaml::Mapping;
use super::*;
use crate::{
CustomWeight, FormattedTable, FormattedTextStyles, LineCount, compute_formatted_text_delta,
};
use super::*;
// Simple transformer to make testing easier.
fn test_parse_markdown(source: &str) -> Vec<FormattedTextLine> {
parse_all(source, |input| parse_markdown_internal(input, false))
@@ -892,6 +889,82 @@ fn test_multi_parse_url() {
);
}
#[test]
fn test_autolink_with_escaped_dot() {
// After a save cycle, `parse_url` may see `https://google\.com` (with `\.` from
// the serializer's escape of `.`). The URL should be unescaped so the buffer
// stores the clean string and the round-trip is stable.
assert_eq!(
test_parse_markdown("https://google\\.com"),
vec![FormattedTextLine::Line(vec![
FormattedTextFragment::hyperlink("https://google.com", "https://google.com"),
])]
);
}
#[test]
fn test_autolink_with_escaped_dash() {
assert_eq!(
test_parse_markdown("https://warp\\-dev.com"),
vec![FormattedTextLine::Line(vec![
FormattedTextFragment::hyperlink("https://warp-dev.com", "https://warp-dev.com"),
])]
);
}
#[test]
fn test_autolink_with_multiple_escapes() {
// Multiple escaped characters (`.`, `-`, `#`) should all be unescaped.
assert_eq!(
test_parse_markdown("https://github\\.com/foo\\-bar\\#anchor"),
vec![FormattedTextLine::Line(vec![
FormattedTextFragment::hyperlink(
"https://github.com/foo-bar#anchor",
"https://github.com/foo-bar#anchor"
),
])]
);
}
#[test]
fn test_autolink_in_link_label_with_escapes() {
// When an escaped URL appears as the display text of an explicit link
// (the `can_autolink = false` path), it should also be unescaped.
let source = "[https://google\\.com](https://google.com)";
assert_eq!(
test_parse_markdown(source),
vec![FormattedTextLine::Line(vec![
FormattedTextFragment::hyperlink("https://google.com", "https://google.com"),
])]
);
}
#[test]
fn test_autolink_escape_roundtrip_is_stable() {
// Parsing a URL with no escapes should produce the same result as
// parsing the same URL with `\.` in place of `.` (simulating a save cycle).
// Both should yield the clean, unescaped URL.
let clean = test_parse_markdown("https://google.com");
let escaped = test_parse_markdown("https://google\\.com");
assert_eq!(clean, escaped);
}
#[test]
fn test_parse_url_preserves_non_escaped_backslash() {
// A backslash followed by a non-punctuation character is not a markdown escape
// sequence and should be left as-is.
let source = "https://example.com/path\\nname";
assert_eq!(
test_parse_markdown(source),
vec![FormattedTextLine::Line(vec![
FormattedTextFragment::hyperlink(
"https://example.com/path\\nname",
"https://example.com/path\\nname"
),
])]
);
}
#[test]
fn test_parse_url_embedded() {
let source = "This iswww.google.comabc";
@@ -954,6 +1027,29 @@ fn test_parse_unclosed_strikethrough() {
);
}
#[test]
fn test_long_delimiter_run_does_not_overflow() {
// Regression test: a run of 256+ identical delimiters used to overflow the
// `u8` run counter, panicking in debug builds and silently wrapping the
// count to 0 (dropping the characters) in release builds. The count is now
// a `usize`, so a long unmatched run must round-trip to literal text.
for delimiter in ["*", "_", "~"] {
// 255 was already fine; 256 is the first value that overflowed `u8`.
for len in [255, 256, 512] {
let source = delimiter.repeat(len);
let parsed = parse_markdown(&source)
.unwrap_or_else(|_| panic!("{len} '{delimiter}' delimiters should parse"));
// An unmatched run carries no content to emphasize, so every
// character survives as literal text.
assert_eq!(
parsed.raw_text(),
format!("{source}\n"),
"{len} '{delimiter}' delimiters must round-trip without loss"
);
}
}
}
#[test]
fn test_parse_escapes() {
let source = "This is \\*not\\* italic. *This* is marked by \\* though";