use anyhow::Result; use serde_yaml::{Mapping, Value}; use std::{ cell::Cell, collections::{HashMap, VecDeque}, rc::Rc, }; use html5ever::{ Attribute, ParseOpts, parse_document, tendril::TendrilSink, tree_builder::TreeBuilderOpts, }; use markup5ever_rcdom::{Node, NodeData, RcDom}; use crate::{ CodeBlockText, FormattedIndentTextInline, FormattedTaskList, FormattedText, FormattedTextFragment, FormattedTextHeader, FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink, OrderedFormattedIndentTextInline, markdown_parser::RUNNABLE_BLOCK_MARKDOWN_LANG, weight::CustomWeight, }; // Top element element tags we are not parsing for right now. // Note that we have "" here because GDocs always include a top level element to add additional // GDocs specific meta-data for its rich text content. const TOP_LEVEL_ELEMENT_TAGS_TO_SKIP: &[&str] = &[ "head", "body", "html", "meta", "table", "b", "div", "ul", "ol", "li", "input", ]; const PHRASING_ELEMENT_TAGS: &[&str] = &[ "span", "i", "code", "strong", "em", "br", "a", "s", "u", "ins", ]; pub const WARP_EMBED_ATTRIBUTE_NAME: &str = "data-warp-embedded-item"; #[derive(Clone, Debug, PartialEq, Eq)] struct ListArg { indent_level: usize, item_type: ListType, start_number: Rc>>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ListType { Checkbox(bool), ListItem { ordered: bool }, } #[derive(Clone, Default)] struct Styling { bold: bool, italic: bool, underline: bool, strikethrough: bool, inline_code: bool, link: Option, } impl Styling { fn update_with_attributes(&mut self, attributes: &[Attribute]) { for attribute in attributes { let attribute_name = attribute.name.local.to_string(); if attribute_name == "style" { let attribute_value = attribute.value.to_string(); let style_dict = parse_style_into_dict(attribute_value.as_str()); if let Some(font_style) = style_dict.get("font-style") { self.italic = *font_style == "italic"; } if let Some(text_decoration) = style_dict.get("text-decoration") { // `text-decoration` can be used for multiple things. If we are missing line-through // here, don't unset the strikethrough. if *text_decoration == "line-through" { self.strikethrough = true; } if *text_decoration == "underline" { self.underline = true; } } if let Some(font_weight) = style_dict.get("font-weight") { if *font_weight == "bold" || *font_weight == "bolder" { self.bold = true; } else { let maybe_integer = font_weight.parse::(); if let Ok(weight_value) = maybe_integer { self.bold = weight_value > 400; } } } // Note that there is not a definitive way in HTML to represent inline code. // Different text editors all use different ways to represent inline code and are // not always compatible to each other, here we chose Notion's way of representing // inline code which is a non-transparent background. We could revisit this in the future. if let Some(color) = style_dict.get("background") { self.inline_code = *color != "transparent"; } } else if attribute_name == "href" { let attribute_value = attribute.value.to_string(); self.link = Some(attribute_value); } } } } /// Find an attribute by name, if it's present. fn get_attribute<'a>(attributes: &'a [Attribute], name: &str) -> Option<&'a str> { attributes.iter().find_map(|attribute| { if &attribute.name.local == name { Some(attribute.value.as_ref()) } else { None } }) } fn includes_attribute(attributes: &[Attribute], name: &str) -> bool { get_attribute(attributes, name).is_some() } fn type_matches(attributes: &[Attribute], value: &str) -> bool { get_attribute(attributes, "type") == Some(value) } // Top-level function to parse a HTML string into a FormattedText document. pub fn parse_html(html: &str) -> Result { let opts = ParseOpts { tree_builder: TreeBuilderOpts { drop_doctype: true, ..Default::default() }, ..Default::default() }; // Parse the document into a Dom element tree. let dom = parse_document(RcDom::default(), opts) .from_utf8() .read_from(&mut html.as_bytes())?; let mut result = VecDeque::new(); // Top-level nodes to visit. let mut nodes: Vec<(Rc, Option)> = Vec::new(); nodes.push((Rc::clone(&dom.document), None)); let mut pending_inline_nodes = Vec::new(); // Active indent level decorating the current node. let mut last_active_indent_level: Option = None; while let Some((node, mut indent_level)) = nodes.pop() { // If the indentation level has changed, we should push all pending inline nodes into the previous // indentation level first. if last_active_indent_level != indent_level && !pending_inline_nodes.is_empty() { if let Some(parsed_node) = parse_pending_inline_nodes(&pending_inline_nodes, last_active_indent_level.as_ref()) { result.push_back(parsed_node); } pending_inline_nodes.clear(); } match &node.data { // Nodes we are not processing. Just push its children into the visit queue. NodeData::Document | NodeData::Doctype { .. } | NodeData::ProcessingInstruction { .. } | NodeData::Comment { .. } => { for child in node.children.borrow().iter().rev() { nodes.push((Rc::clone(child), indent_level.clone())); } } // If we observe plain text in the top level nodes. Add them as plain text lines. NodeData::Text { contents } => { if !contents.borrow().trim().is_empty() { pending_inline_nodes.push(node); } } NodeData::Element { name, attrs, .. } => { let node_name = name.local.to_string(); let mut decorated_styling = Styling::default(); // Confluence does not follow the common pattern of marking code blocks in
 specifically in its view mode.
                // Instead, it marks code blocks in  with a specific attribute.
                let is_confluence_code_block = node_name.as_str() == "span"
                    && includes_attribute(&attrs.borrow(), "data-ds--code--code-block");

                // If the node is an element we are skip processing, push its children into the visit queue and skip
                // to the next iteration.
                if TOP_LEVEL_ELEMENT_TAGS_TO_SKIP.contains(&node_name.as_str()) {
                    let is_unordered_list = node_name.as_str() == "ul";
                    let is_ordered_list = node_name.as_str() == "ol";
                    let start_number = if is_ordered_list {
                        get_attribute(&attrs.borrow(), "start").and_then(|value| value.parse().ok())
                    } else {
                        None
                    };
                    if is_unordered_list || is_ordered_list {
                        let level = match &indent_level {
                            Some(level) => level.indent_level + 1,
                            _ => 0,
                        };
                        indent_level = Some(ListArg {
                            item_type: ListType::ListItem {
                                ordered: is_ordered_list,
                            },
                            start_number: Rc::new(Cell::new(start_number)),
                            indent_level: level,
                        });
                    };

                    // Check if the first node represents input.
                    if let Some(NodeData::Element {
                        name,
                        attrs: child_node_attr,
                        ..
                    }) = node.children.borrow().iter().next().map(|node| &node.data)
                    {
                        let child_node_name = name.local.to_string();
                        if let Some(indent_arg) = &mut indent_level
                            && child_node_name.as_str() == "input"
                            && type_matches(&child_node_attr.borrow(), "checkbox")
                        {
                            indent_arg.item_type = ListType::Checkbox(includes_attribute(
                                &child_node_attr.borrow(),
                                "checked",
                            ));
                        }
                    }

                    for child in node.children.borrow().iter().rev() {
                        nodes.push((Rc::clone(child), indent_level.clone()));
                    }

                    if !pending_inline_nodes.is_empty() {
                        if let Some(parsed_nodes) = parse_pending_inline_nodes(
                            &pending_inline_nodes,
                            last_active_indent_level.as_ref(),
                        ) {
                            result.push_back(parsed_nodes);
                        }
                        pending_inline_nodes.clear();
                    }

                    last_active_indent_level = indent_level;
                    continue;
                } else if PHRASING_ELEMENT_TAGS.contains(&node_name.as_str())
                    && !is_confluence_code_block
                {
                    pending_inline_nodes.push(node);
                    last_active_indent_level = indent_level;
                    continue;
                }

                if !pending_inline_nodes.is_empty() {
                    if let Some(parsed_nodes) = parse_pending_inline_nodes(
                        &pending_inline_nodes,
                        last_active_indent_level.as_ref(),
                    ) {
                        result.push_back(parsed_nodes);
                    }
                    pending_inline_nodes.clear();
                }

                // Update styling based on the node's attribute.
                decorated_styling.update_with_attributes(&attrs.borrow());

                result.push_back(match node_name.as_str() {
                    // If it's a code block, process its children node as plain text.
                    "pre" => {
                        if let Some(val) = get_attribute(&attrs.borrow(), WARP_EMBED_ATTRIBUTE_NAME)
                        {
                            FormattedTextLine::Embedded(Mapping::from_iter([(
                                Value::String("id".to_string()),
                                Value::String(val.to_string()),
                            )]))
                        } else {
                            // TODO: Support Github's code block representation.
                            let (content, language) =
                                parse_code_block_and_language(&node.children.borrow());
                            FormattedTextLine::CodeBlock(CodeBlockText {
                                lang: language.unwrap_or(RUNNABLE_BLOCK_MARKDOWN_LANG.to_string()),
                                code: content,
                            })
                        }
                    }
                    "span" if is_confluence_code_block => {
                        FormattedTextLine::CodeBlock(CodeBlockText {
                            lang: get_attribute(&attrs.borrow(), "data-code-lang")
                                .unwrap_or(RUNNABLE_BLOCK_MARKDOWN_LANG)
                                .to_string(),
                            code: parse_text_only(&node.children.borrow()),
                        })
                    }
                    "h1" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 1,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "h2" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 2,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "h3" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 3,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "h4" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 4,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "h5" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 5,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "h6" => FormattedTextLine::Heading(FormattedTextHeader {
                        heading_size: 6,
                        text: parse_phrasing_content(
                            &node.children.borrow(),
                            decorated_styling.clone(),
                        ),
                    }),
                    "br" => FormattedTextLine::LineBreak,
                    "hr" => FormattedTextLine::HorizontalRule,
                    _ => {
                        // Take into consideration the indent level when parsing the nodes.
                        let parsed_node = parse_pending_inline_nodes(
                            &node.children.borrow(),
                            indent_level.as_ref(),
                        );

                        match parsed_node {
                            Some(node) => node,
                            None => FormattedTextLine::Line(Vec::new()),
                        }
                    }
                })
            }
        }

        last_active_indent_level = indent_level;
    }

    if !pending_inline_nodes.is_empty() {
        if let Some(parsed_nodes) =
            parse_pending_inline_nodes(&pending_inline_nodes, last_active_indent_level.as_ref())
        {
            result.push_back(parsed_nodes);
        }
        pending_inline_nodes.clear();
    }

    Ok(FormattedText { lines: result })
}

// Push all pending inline nodes into the result. Take into consideration the active indent level.
fn parse_pending_inline_nodes(
    nodes: &[Rc],
    last_active_indent_level: Option<&ListArg>,
) -> Option {
    let internal = parse_phrasing_content(nodes, Default::default());

    if !internal.is_empty() {
        Some(match last_active_indent_level {
            Some(list) => match list.item_type {
                ListType::ListItem { ordered: true } => {
                    FormattedTextLine::OrderedList(OrderedFormattedIndentTextInline {
                        // Take the start number, so that it's only applied to the first item in
                        // the list.
                        number: list.start_number.take(),
                        indented_text: FormattedIndentTextInline {
                            indent_level: list.indent_level,
                            text: internal,
                        },
                    })
                }
                ListType::ListItem { ordered: false } => {
                    FormattedTextLine::UnorderedList(FormattedIndentTextInline {
                        indent_level: list.indent_level,
                        text: internal,
                    })
                }
                ListType::Checkbox(checked) => FormattedTextLine::TaskList(FormattedTaskList {
                    complete: checked,
                    indent_level: list.indent_level,
                    text: internal,
                }),
            },
            None => FormattedTextLine::Line(internal),
        })
    } else {
        None
    }
}

// Parse the phrasing content: https://developer.mozilla.org/en-US/docs/Web/HTML/Content_categories#phrasing_content
// into an inline formatted text.
fn parse_phrasing_content(nodes: &[Rc], text_styling: Styling) -> FormattedTextInline {
    let mut result = Vec::new();
    for node in nodes {
        if is_spacing_span(node) {
            result.push(phrasing_to_formatted_text(" ", &text_styling));
            continue;
        }

        match &node.data {
            // We should not observe these in the phrasing content.
            NodeData::Document
            | NodeData::Doctype { .. }
            | NodeData::ProcessingInstruction { .. }
            | NodeData::Comment { .. } => {}
            // Push text fragment based on the current text_styling.
            NodeData::Text { contents } => {
                let content = contents.borrow().trim_end_matches('\n').to_string();

                // Filter out singular empty lines after trimming as they are from HTML formatting and shouldn't
                // be inserted into the content. The rare valid case we might be missing here is 

\n

. // But all major rich text editors I have tested use
to represent it instead. if content.is_empty() { continue; } result.push(phrasing_to_formatted_text(content, &text_styling)); } NodeData::Element { name, attrs, .. } => { let node_name = name.local.to_string(); let mut decorated_styling = text_styling.clone(); decorated_styling.update_with_attributes(&attrs.borrow()); match node_name.as_ref() { "b" | "strong" => decorated_styling.bold = true, "i" | "em" => decorated_styling.italic = true, "s" => decorated_styling.strikethrough = true, "u" | "ins" => decorated_styling.underline = true, "code" => decorated_styling.inline_code = true, // TODO: We need to add more phrasing styling we support (e.g. links) here. // https://linear.app/warpdotdev/issue/CLD-335/add-html-parsing-for-headers-and-lists _ => (), }; result.extend(parse_phrasing_content( node.children.borrow().as_ref(), decorated_styling, )); } } } result } /// Converts styled phrasing text to a fragment of formatted text. fn phrasing_to_formatted_text(text: impl Into, styling: &Styling) -> FormattedTextFragment { let weight = if styling.bold { Some(CustomWeight::Bold) } else { None }; FormattedTextFragment { text: text.into(), styles: FormattedTextStyles { weight, italic: styling.italic, underline: styling.underline, strikethrough: styling.strikethrough, hyperlink: styling.link.clone().map(Hyperlink::Url), inline_code: styling.inline_code, }, } } /// Chrome and Safari replace some spaces in copied content with non-breaking spaces. This matches /// such spaces. /// /// See [this ProseMirror thread](https://discuss.prosemirror.net/t/non-breaking-spaces-being-added-to-pasted-html/3911/4). fn is_spacing_span(node: &Rc) -> bool { if let NodeData::Element { name, attrs, .. } = &node.data { if &name.local != "span" { return false; } // The span must have either no attrs or a single `class="Apple-converted-space"` class. let attrs = attrs.borrow(); if attrs.len() > 1 { return false; } let css_class = attrs.first().filter(|attr| &attr.name.local == "class"); if css_class.is_some_and(|class| &*class.value != "Apple-converted-space") { return false; } let content = node.children.borrow(); if content.len() != 1 { return false; } match &content[0].data { NodeData::Text { contents } => &**contents.borrow() == "\u{00a0}", _ => false, } } else { false } } // Only parse out text content in the provided nodes and their children. fn parse_text_only(nodes: &[Rc]) -> String { let mut text = String::new(); for node in nodes { if let NodeData::Text { contents } = &node.data { text.push_str(contents.borrow().to_string().as_str()); } else { text.push_str(&parse_text_only(&node.children.borrow())); } } text } // Recursively parse out the code block content and it's language info. fn parse_code_block_and_language(nodes: &[Rc]) -> (String, Option) { let mut text = String::new(); let mut language = None; for node in nodes { match &node.data { NodeData::Text { contents } => text.push_str(contents.borrow().to_string().as_str()), NodeData::Element { name, attrs, .. } => { let node_name = name.local.to_string(); if node_name == "code" && let Some(parsed_lang) = get_attribute(&attrs.borrow(), "class") .and_then(|s| s.strip_prefix("language-").map(|result| result.to_string())) { language = Some(parsed_lang); } let (child_str, new_lang) = parse_code_block_and_language(&node.children.borrow()); text.push_str(&child_str); if new_lang.is_some() { language = new_lang; } } _ => text.push_str(&parse_text_only(&node.children.borrow())), } } (text, language) } // Parse a HMTL style string into its corresponding name -> value hashmap // For example "font-style:italic;font-weight:400" will be parsed into // {"font-style": "italic", "font-weight": "400"}. fn parse_style_into_dict(style: &str) -> HashMap<&str, &str> { let style_pairs = style.split(';'); let mut style_dict = HashMap::new(); for pair in style_pairs { if pair.contains(':') { let name_and_value: Vec<&str> = pair.split(':').collect(); style_dict.insert(name_and_value[0].trim(), name_and_value[1].trim()); } } style_dict } #[cfg(test)] #[path = "html_parser_test.rs"] mod tests;