first pass of merging in warp (doesn't build)
This commit is contained in:
Ryan Ward
@@ -0,0 +1,401 @@
|
||||
//! Parses `.ipynb` (Jupyter) notebooks directly into the [`FormattedText`]
|
||||
//! representation that Warp's rich-text/notebook renderer consumes.
|
||||
//!
|
||||
//! This is **render-only**: it produces a read-only view of the notebook's
|
||||
//! existing content (markdown cells, code cells, and saved outputs). It does
|
||||
//! not execute cells or round-trip edits back to the file.
|
||||
//!
|
||||
//! Only nbformat v4 is supported. Anything that fails to parse as a v4 notebook
|
||||
//! returns an [`IpynbError`] so callers can fall back to showing the raw file
|
||||
//! contents instead of a blank view.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use markdown_parser::{
|
||||
CodeBlockText, FormattedImage, FormattedText, FormattedTextFragment, FormattedTextLine,
|
||||
parse_markdown, parse_markdown_with_gfm_tables,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
/// The only nbformat major version this converter understands.
|
||||
const SUPPORTED_NBFORMAT: i64 = 4;
|
||||
|
||||
/// Maximum length of a code-block language tag we will emit. Real language
|
||||
/// names are short; a longer value is treated as untrusted junk and dropped so
|
||||
/// it cannot bloat every code block.
|
||||
const MAX_LANGUAGE_TAG_CHARS: usize = 32;
|
||||
|
||||
/// Error produced when the input cannot be rendered as a supported notebook.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum IpynbError {
|
||||
/// The input was not valid notebook JSON.
|
||||
#[error("failed to parse notebook JSON: {0}")]
|
||||
Parse(#[from] serde_json::Error),
|
||||
/// The notebook used an unsupported nbformat version (only v4 is supported).
|
||||
#[error(
|
||||
"unsupported notebook format: nbformat={nbformat:?} (only v{SUPPORTED_NBFORMAT} is supported)"
|
||||
)]
|
||||
UnsupportedFormat { nbformat: Option<i64> },
|
||||
}
|
||||
|
||||
/// Convert the JSON contents of a `.ipynb` file into [`FormattedText`].
|
||||
///
|
||||
/// `gfm_tables` selects the GFM-table-aware Markdown parser for markdown cells,
|
||||
/// mirroring the `Buffer::from_markdown` behavior (the caller passes the
|
||||
/// `MarkdownTables` feature flag state).
|
||||
///
|
||||
/// Returns an [`IpynbError`] if the input is not a parseable nbformat v4
|
||||
/// notebook; callers should fall back to [`raw_fallback_formatted_text`] in that
|
||||
/// case so the contents are shown verbatim (never a blank view).
|
||||
pub fn ipynb_to_formatted_text(json: &str, gfm_tables: bool) -> Result<FormattedText, IpynbError> {
|
||||
let notebook: Notebook = serde_json::from_str(json)?;
|
||||
|
||||
// Guard against arbitrary JSON that happens to deserialize into an empty
|
||||
// notebook: require an explicit, supported nbformat version.
|
||||
if notebook.nbformat != Some(SUPPORTED_NBFORMAT) {
|
||||
return Err(IpynbError::UnsupportedFormat {
|
||||
nbformat: notebook.nbformat,
|
||||
});
|
||||
}
|
||||
|
||||
let language = notebook.language();
|
||||
// Convert each cell independently (a 1:1 map with an exact size hint), then
|
||||
// size the final buffer from the per-cell line counts so it is allocated
|
||||
// exactly once instead of growing incrementally.
|
||||
let per_cell: Vec<Vec<FormattedTextLine>> = notebook
|
||||
.cells
|
||||
.iter()
|
||||
.map(|cell| cell_lines(cell, &language, gfm_tables))
|
||||
.collect();
|
||||
let total_lines = per_cell.iter().map(Vec::len).sum();
|
||||
let mut lines = Vec::with_capacity(total_lines);
|
||||
for lines_for_cell in per_cell {
|
||||
lines.extend(lines_for_cell);
|
||||
}
|
||||
|
||||
Ok(FormattedText::new_trimmed(lines))
|
||||
}
|
||||
|
||||
/// Build a verbatim [`FormattedText`] fallback for content that is not a
|
||||
/// parseable notebook (malformed JSON, unsupported nbformat version, etc.). The
|
||||
/// raw content is placed in a single code block so any Markdown/HTML inside it
|
||||
/// is shown verbatim, never re-interpreted.
|
||||
pub fn raw_fallback_formatted_text(content: &str) -> FormattedText {
|
||||
FormattedText::new_trimmed(vec![FormattedTextLine::CodeBlock(CodeBlockText {
|
||||
lang: "json".to_string(),
|
||||
code: content.trim_end_matches('\n').to_string(),
|
||||
})])
|
||||
}
|
||||
|
||||
/// Convert a single notebook cell into its formatted-text lines.
|
||||
fn cell_lines(cell: &Cell, language: &str, gfm_tables: bool) -> Vec<FormattedTextLine> {
|
||||
match cell.cell_type.as_str() {
|
||||
"markdown" => {
|
||||
let source = cell.source.to_text();
|
||||
markdown_block_lines(source.trim_end_matches('\n'), gfm_tables)
|
||||
}
|
||||
"code" => {
|
||||
let source = cell.source.to_text();
|
||||
let mut lines = code_block_lines(language, source.trim_end_matches('\n'));
|
||||
lines.extend(cell.outputs.iter().flat_map(output_lines));
|
||||
lines
|
||||
}
|
||||
// Raw cells are passed through verbatim in Jupyter; render them as a
|
||||
// plain (unhighlighted) code block so their contents can't inject
|
||||
// unexpected markdown.
|
||||
"raw" => {
|
||||
let source = cell.source.to_text();
|
||||
code_block_lines("", source.trim_end_matches('\n'))
|
||||
}
|
||||
// Unknown / future cell types are skipped rather than rendered raw.
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A markdown cell, parsed into formatted text and separated from surrounding
|
||||
/// blocks by a line break. Empty cells produce no lines.
|
||||
fn markdown_block_lines(content: &str, gfm_tables: bool) -> Vec<FormattedTextLine> {
|
||||
if content.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let parse_fn = if gfm_tables {
|
||||
parse_markdown_with_gfm_tables
|
||||
} else {
|
||||
parse_markdown
|
||||
};
|
||||
// A markdown cell is, by definition, Markdown; parse it once. If parsing
|
||||
// somehow fails, preserve the content verbatim as a plain line rather than
|
||||
// dropping it.
|
||||
let parsed = parse_fn(content).unwrap_or_else(|_| {
|
||||
FormattedText::new(vec![FormattedTextLine::Line(vec![
|
||||
FormattedTextFragment::plain_text(content),
|
||||
])])
|
||||
});
|
||||
let mut lines = Vec::from(parsed.lines);
|
||||
lines.push(FormattedTextLine::LineBreak);
|
||||
lines
|
||||
}
|
||||
|
||||
/// A code block with the given language tag (empty for none), separated from
|
||||
/// surrounding blocks by a line break.
|
||||
fn code_block_lines(language: &str, content: &str) -> Vec<FormattedTextLine> {
|
||||
vec![
|
||||
FormattedTextLine::CodeBlock(CodeBlockText {
|
||||
lang: language.to_string(),
|
||||
code: content.to_string(),
|
||||
}),
|
||||
FormattedTextLine::LineBreak,
|
||||
]
|
||||
}
|
||||
|
||||
/// The lines for a single saved cell output. Skipped outputs produce no lines.
|
||||
fn output_lines(output: &Output) -> Vec<FormattedTextLine> {
|
||||
match output.output_type.as_str() {
|
||||
"stream" => match &output.text {
|
||||
Some(text) => text_output_lines(&text.to_text()),
|
||||
None => Vec::new(),
|
||||
},
|
||||
"execute_result" | "display_data" => {
|
||||
let Some(data) = &output.data else {
|
||||
return Vec::new();
|
||||
};
|
||||
// Prefer images, then plain text. Other MIME types (text/html,
|
||||
// LaTeX, widgets, ...) are intentionally skipped in v1.
|
||||
if let Some(value) = data.get("image/png") {
|
||||
image_lines("image/png", value)
|
||||
} else if let Some(value) = data.get("image/jpeg") {
|
||||
image_lines("image/jpeg", value)
|
||||
} else if let Some(value) = data.get("text/plain") {
|
||||
text_output_lines(&value_to_text(value))
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
"error" => {
|
||||
let traceback = output
|
||||
.traceback
|
||||
.as_ref()
|
||||
.map(|tb| tb.join("\n"))
|
||||
.unwrap_or_default();
|
||||
// ANSI escapes (common in colored tracebacks) are stripped centrally
|
||||
// by `text_output_lines`.
|
||||
text_output_lines(&traceback)
|
||||
}
|
||||
// Unknown output types are skipped.
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A text output as a plain (unhighlighted) code block. Empty output produces
|
||||
/// no lines.
|
||||
///
|
||||
/// TODO: Bounding pathologically large outputs is left to
|
||||
/// a future change that models truncation without polluting buffer content
|
||||
/// (e.g. a "show more" affordance) rather than injecting placeholder text.
|
||||
fn text_output_lines(text: &str) -> Vec<FormattedTextLine> {
|
||||
// TODO: Remove ANSI stripping once we support ANSI-color rendering (a color
|
||||
// attribute on `FormattedTextStyles` + SGR parsing)
|
||||
let stripped = strip_ansi(text);
|
||||
let text = stripped.trim_end_matches('\n');
|
||||
if text.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
code_block_lines("", text)
|
||||
}
|
||||
|
||||
/// An embedded image output as a base64 data-URI image. Empty payloads produce
|
||||
/// no lines.
|
||||
fn image_lines(mime: &str, value: &serde_json::Value) -> Vec<FormattedTextLine> {
|
||||
let base64: String = value_to_text(value)
|
||||
.chars()
|
||||
.filter(|c| !c.is_whitespace())
|
||||
.collect();
|
||||
if base64.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
vec![
|
||||
FormattedTextLine::Image(FormattedImage {
|
||||
alt_text: "output".to_string(),
|
||||
source: format!("data:{mime};base64,{base64}"),
|
||||
title: None,
|
||||
}),
|
||||
FormattedTextLine::LineBreak,
|
||||
]
|
||||
}
|
||||
|
||||
/// Strip ANSI escape sequences (CSI/SGR colors, OSC, and simple escapes) so
|
||||
/// tracebacks render as readable plain text.
|
||||
fn strip_ansi(input: &str) -> String {
|
||||
let mut out = String::with_capacity(input.len());
|
||||
let mut chars = input.chars().peekable();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch != '\u{1b}' {
|
||||
out.push(ch);
|
||||
continue;
|
||||
}
|
||||
match chars.peek() {
|
||||
// CSI sequence: ESC [ ... <final byte 0x40-0x7E>
|
||||
Some('[') => {
|
||||
chars.next();
|
||||
for next in chars.by_ref() {
|
||||
if ('\u{40}'..='\u{7e}').contains(&next) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// OSC sequence: ESC ] ... terminated by BEL or ST (ESC \)
|
||||
Some(']') => {
|
||||
chars.next();
|
||||
while let Some(&next) = chars.peek() {
|
||||
if next == '\u{07}' {
|
||||
chars.next();
|
||||
break;
|
||||
}
|
||||
if next == '\u{1b}' {
|
||||
chars.next();
|
||||
if chars.peek() == Some(&'\\') {
|
||||
chars.next();
|
||||
}
|
||||
break;
|
||||
}
|
||||
chars.next();
|
||||
}
|
||||
}
|
||||
// Other escapes (e.g. ESC ( B): drop the single following byte.
|
||||
Some(_) => {
|
||||
chars.next();
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Convert a JSON value that is either a string or an array of strings into a
|
||||
/// single string (notebook source and text fields use both forms).
|
||||
fn value_to_text(value: &serde_json::Value) -> String {
|
||||
match value {
|
||||
serde_json::Value::String(s) => s.clone(),
|
||||
serde_json::Value::Array(items) => items.iter().filter_map(|v| v.as_str()).collect(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize a notebook-declared language into a safe code-block language tag.
|
||||
fn sanitize_language(raw: &str) -> String {
|
||||
let trimmed = raw.trim();
|
||||
let is_safe = !trimmed.is_empty()
|
||||
&& trimmed.chars().count() <= MAX_LANGUAGE_TAG_CHARS
|
||||
&& trimmed
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '#' | '-' | '_' | '.'));
|
||||
if is_safe {
|
||||
trimmed.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Top-level notebook structure (nbformat v4, only the fields we render).
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Notebook {
|
||||
#[serde(default)]
|
||||
nbformat: Option<i64>,
|
||||
cells: Vec<Cell>,
|
||||
#[serde(default)]
|
||||
metadata: Metadata,
|
||||
}
|
||||
|
||||
impl Notebook {
|
||||
/// The code-block language, derived from notebook metadata and sanitized to
|
||||
/// a safe tag (see [`sanitize_language`]). Empty if the notebook does not
|
||||
/// declare a language, or declares one that is not a safe identifier.
|
||||
fn language(&self) -> String {
|
||||
let raw = self
|
||||
.metadata
|
||||
.language_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.name.clone())
|
||||
.or_else(|| {
|
||||
self.metadata
|
||||
.kernelspec
|
||||
.as_ref()
|
||||
.and_then(|spec| spec.language.clone())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
sanitize_language(&raw)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Metadata {
|
||||
#[serde(default)]
|
||||
language_info: Option<LanguageInfo>,
|
||||
#[serde(default)]
|
||||
kernelspec: Option<Kernelspec>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LanguageInfo {
|
||||
#[serde(default)]
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Kernelspec {
|
||||
#[serde(default)]
|
||||
language: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Cell {
|
||||
#[serde(default)]
|
||||
cell_type: String,
|
||||
#[serde(default)]
|
||||
source: Source,
|
||||
#[serde(default)]
|
||||
outputs: Vec<Output>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Output {
|
||||
#[serde(default)]
|
||||
output_type: String,
|
||||
/// Present for `stream` outputs.
|
||||
#[serde(default)]
|
||||
text: Option<Source>,
|
||||
/// Present for `execute_result` / `display_data` outputs (MIME -> value).
|
||||
#[serde(default)]
|
||||
data: Option<BTreeMap<String, serde_json::Value>>,
|
||||
/// Present for `error` outputs.
|
||||
#[serde(default)]
|
||||
traceback: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// A notebook `source`/`text` field, which may be a single string or a list of
|
||||
/// strings (each typically including its trailing newline).
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Source {
|
||||
Lines(Vec<String>),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl Default for Source {
|
||||
fn default() -> Self {
|
||||
Source::Text(String::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl Source {
|
||||
fn to_text(&self) -> String {
|
||||
match self {
|
||||
Source::Lines(lines) => lines.concat(),
|
||||
Source::Text(text) => text.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "lib_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,413 @@
|
||||
use markdown_parser::{CodeBlockText, FormattedImage, FormattedText, FormattedTextLine};
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Convert a notebook, asserting it parses successfully (GFM tables off).
|
||||
fn convert(json: &str) -> FormattedText {
|
||||
ipynb_to_formatted_text(json, false).expect("should convert")
|
||||
}
|
||||
|
||||
/// All code blocks in the formatted text, in order.
|
||||
fn code_blocks(ft: &FormattedText) -> Vec<&CodeBlockText> {
|
||||
ft.lines
|
||||
.iter()
|
||||
.filter_map(|line| match line {
|
||||
FormattedTextLine::CodeBlock(block) => Some(block),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All images in the formatted text, in order.
|
||||
fn images(ft: &FormattedText) -> Vec<&FormattedImage> {
|
||||
ft.lines
|
||||
.iter()
|
||||
.filter_map(|line| match line {
|
||||
FormattedTextLine::Image(image) => Some(image),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_and_code_cells() {
|
||||
let json = r##"{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5,
|
||||
"metadata": {"language_info": {"name": "python"}},
|
||||
"cells": [
|
||||
{"cell_type": "markdown", "source": ["# Title\n", "Some text"]},
|
||||
{"cell_type": "code", "source": "print('hi')", "outputs": []}
|
||||
]
|
||||
}"##;
|
||||
|
||||
let ft = convert(json);
|
||||
// The markdown cell is parsed once into formatted text...
|
||||
let raw = ft.raw_text();
|
||||
assert!(raw.contains("Title"), "got: {raw:?}");
|
||||
assert!(raw.contains("Some text"), "got: {raw:?}");
|
||||
// ...and the code cell becomes a structured code block (no fence/re-parse).
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].lang, "python");
|
||||
assert_eq!(blocks[0].code, "print('hi')");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_cell_without_language() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "x = 1"}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let blocks_owned = convert(json);
|
||||
let blocks = code_blocks(&blocks_owned);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
// No language tag when the notebook does not declare one.
|
||||
assert_eq!(blocks[0].lang, "");
|
||||
assert_eq!(blocks[0].code, "x = 1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_language_falls_back_to_kernelspec() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"metadata": {"kernelspec": {"language": "julia"}},
|
||||
"cells": [{"cell_type": "code", "source": "1 + 1"}]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
assert_eq!(code_blocks(&ft)[0].lang, "julia");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_language_with_backticks_is_sanitized() {
|
||||
// A hostile language tag containing backticks (and a newline) must never end
|
||||
// up as a code block's language. Because the language is now a struct field
|
||||
// (not a fence), this can't break rendering regardless; we still drop it.
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"metadata": {"language_info": {"name": "py```\ninjected"}},
|
||||
"cells": [{"cell_type": "code", "source": "x = 1"}]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks[0].lang, "");
|
||||
assert_eq!(blocks[0].code, "x = 1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_language_with_special_chars_is_preserved() {
|
||||
// Legitimate language names containing `+`/`#`/`-` are kept verbatim.
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"metadata": {"language_info": {"name": "c++"}},
|
||||
"cells": [{"cell_type": "code", "source": "int x;"}]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
assert_eq!(code_blocks(&ft)[0].lang, "c++");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_language_accepts_and_rejects() {
|
||||
// Identifier-like tokens (including the punctuation real language names use)
|
||||
// are accepted and trimmed.
|
||||
assert_eq!(sanitize_language("python"), "python");
|
||||
assert_eq!(sanitize_language("C++"), "C++");
|
||||
assert_eq!(sanitize_language("objective-c"), "objective-c");
|
||||
assert_eq!(sanitize_language(" rust "), "rust");
|
||||
// Backticks, whitespace, other info-string syntax, and oversized values are
|
||||
// rejected, yielding an empty (safe) tag.
|
||||
assert_eq!(sanitize_language("py`thon"), "");
|
||||
assert_eq!(sanitize_language("two words"), "");
|
||||
assert_eq!(sanitize_language(""), "");
|
||||
assert_eq!(
|
||||
sanitize_language(&"a".repeat(MAX_LANGUAGE_TAG_CHARS + 1)),
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stream_output_renders_as_text_block() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"metadata": {"language_info": {"name": "python"}},
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "print('hello')", "outputs": [
|
||||
{"output_type": "stream", "name": "stdout", "text": ["hello\n"]}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let blocks = code_blocks(&ft);
|
||||
// Code cell, then its (unhighlighted) text output.
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].lang, "python");
|
||||
assert_eq!(blocks[0].code, "print('hello')");
|
||||
assert_eq!(blocks[1].lang, "");
|
||||
assert_eq!(blocks[1].code, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_execute_result_text_plain() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "2 + 2", "outputs": [
|
||||
{"output_type": "execute_result", "data": {"text/plain": "4"}, "metadata": {}}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[1].code, "4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_traceback_strips_ansi() {
|
||||
// Traceback lines containing SGR color escape sequences.
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "boom", "outputs": [
|
||||
{"output_type": "error", "ename": "NameError", "evalue": "boom",
|
||||
"traceback": ["\u001b[0;31mNameError\u001b[0m: name 'boom'", "is not defined"]}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let raw = ft.raw_text();
|
||||
assert!(
|
||||
!raw.contains('\u{1b}'),
|
||||
"ANSI escapes should be stripped: {raw:?}"
|
||||
);
|
||||
assert!(raw.contains("NameError: name 'boom'"), "got: {raw:?}");
|
||||
assert!(raw.contains("is not defined"), "got: {raw:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_png_output_renders_as_data_uri_image() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "plot()", "outputs": [
|
||||
{"output_type": "display_data", "data": {"image/png": "iVBORw0KGgo=\n"}, "metadata": {}}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let images = images(&ft);
|
||||
assert_eq!(images.len(), 1);
|
||||
// Whitespace in the embedded base64 is stripped.
|
||||
assert_eq!(images[0].source, "data:image/png;base64,iVBORw0KGgo=");
|
||||
assert_eq!(images[0].alt_text, "output");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_payload_is_not_validated_by_parser() {
|
||||
// The parser no longer size-checks or validates base64 payloads: it emits
|
||||
// the `data:` URI verbatim and defers decoding and size limits to the shared
|
||||
// asset layer (mirroring how Markdown `data:` images are handled). An
|
||||
// undecodable payload is still emitted as an image here and simply fails to
|
||||
// load at render time (silent omission).
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "plot()", "outputs": [
|
||||
{"output_type": "display_data", "data": {"image/png": "not-valid-base64"}, "metadata": {}}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let images = images(&ft);
|
||||
assert_eq!(
|
||||
images.len(),
|
||||
1,
|
||||
"payload should be emitted without parser-side validation"
|
||||
);
|
||||
assert_eq!(images[0].source, "data:image/png;base64,not-valid-base64");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_preferred_over_text_plain() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "plot()", "outputs": [
|
||||
{"output_type": "execute_result",
|
||||
"data": {"image/png": "AAAA", "text/plain": "<Figure>"}, "metadata": {}}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
assert_eq!(images(&ft).len(), 1);
|
||||
assert_eq!(images(&ft)[0].source, "data:image/png;base64,AAAA");
|
||||
assert!(
|
||||
!ft.raw_text().contains("<Figure>"),
|
||||
"text/plain should be skipped when an image exists"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unsupported_mime_is_skipped() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "html()", "outputs": [
|
||||
{"output_type": "execute_result", "data": {"text/html": "<b>hi</b>"}, "metadata": {}}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
// The code still renders; the unsupported HTML output is dropped.
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].code, "html()");
|
||||
assert!(images(&ft).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backticks_in_code_are_stored_verbatim() {
|
||||
// Source containing a triple-backtick run is stored verbatim in the code
|
||||
// block; there is no fence to escape, so it cannot break out.
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{"cell_type": "code", "source": "s = \"\"\"```\""}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].code, "s = \"\"\"```\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_notebook_is_ok() {
|
||||
let json = r#"{"nbformat": 4, "cells": []}"#;
|
||||
let ft = convert(json);
|
||||
assert!(
|
||||
ft.lines.is_empty(),
|
||||
"expected no lines, got: {:?}",
|
||||
ft.lines
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_malformed_json_is_error() {
|
||||
let result = ipynb_to_formatted_text("{ not valid json", false);
|
||||
assert!(matches!(result, Err(IpynbError::Parse(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_v4_notebook_is_error() {
|
||||
let json = r#"{"nbformat": 3, "cells": []}"#;
|
||||
let result = ipynb_to_formatted_text(json, false);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(IpynbError::UnsupportedFormat { nbformat: Some(3) })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_missing_nbformat_is_error() {
|
||||
// Arbitrary JSON that lacks an nbformat field must not render as a blank
|
||||
// notebook; it should error so the caller falls back to raw content.
|
||||
let json = r#"{"some": "json", "cells": []}"#;
|
||||
let result = ipynb_to_formatted_text(json, false);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(IpynbError::UnsupportedFormat { nbformat: None })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_missing_cells_is_error() {
|
||||
// A v4 notebook that omits the required `cells` field must error rather than
|
||||
// deserialize as an empty (blank-rendering) notebook, so the caller falls
|
||||
// back to raw content. An explicit `"cells": []` remains a valid empty
|
||||
// notebook (see `test_empty_notebook_is_ok`).
|
||||
let json = r#"{"nbformat": 4}"#;
|
||||
let result = ipynb_to_formatted_text(json, false);
|
||||
assert!(matches!(result, Err(IpynbError::Parse(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_raw_cell_rendered_as_plain_block() {
|
||||
let json = r#"{
|
||||
"nbformat": 4,
|
||||
"metadata": {"language_info": {"name": "python"}},
|
||||
"cells": [
|
||||
{"cell_type": "raw", "source": "raw content"}
|
||||
]
|
||||
}"#;
|
||||
|
||||
let ft = convert(json);
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
// Raw cells are not tagged with the kernel language.
|
||||
assert_eq!(blocks[0].lang, "");
|
||||
assert_eq!(blocks[0].code, "raw content");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_strip_ansi_handles_csi_and_osc() {
|
||||
assert_eq!(strip_ansi("\u{1b}[0;31mred\u{1b}[0m"), "red");
|
||||
assert_eq!(strip_ansi("plain text"), "plain text");
|
||||
// OSC sequence terminated by BEL.
|
||||
assert_eq!(strip_ansi("\u{1b}]0;title\u{07}body"), "body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_text_output_is_preserved_verbatim() {
|
||||
// Large outputs are rendered in full (no arbitrary truncation or synthetic
|
||||
// placeholder text) so select-all/copy yields the canonical content.
|
||||
let big = "a".repeat(250_000);
|
||||
let json = format!(
|
||||
r#"{{"nbformat": 4, "cells": [{{"cell_type": "code", "source": "x", "outputs": [{{"output_type": "stream", "name": "stdout", "text": "{big}"}}]}}]}}"#
|
||||
);
|
||||
|
||||
let ft = convert(&json);
|
||||
let blocks = code_blocks(&ft);
|
||||
let output = &blocks[1].code;
|
||||
assert!(
|
||||
!output.contains("[output truncated]"),
|
||||
"output must not contain a synthetic truncation marker"
|
||||
);
|
||||
assert_eq!(output.chars().count(), big.chars().count());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_raw_fallback_holds_content_verbatim() {
|
||||
// Content that is not a parseable notebook is placed in a single json code
|
||||
// block so any Markdown/HTML inside it is shown verbatim, not interpreted.
|
||||
let raw = "{ \"nbformat\": 4, # Heading <b>bold</b>";
|
||||
let ft = raw_fallback_formatted_text(raw);
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].lang, "json");
|
||||
assert_eq!(blocks[0].code, raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_raw_fallback_handles_backticks_verbatim() {
|
||||
// Raw content containing a triple-backtick run is stored verbatim; there is
|
||||
// no fence for it to break out of.
|
||||
let ft = raw_fallback_formatted_text("```");
|
||||
let blocks = code_blocks(&ft);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].code, "```");
|
||||
}
|
||||
Reference in New Issue
Block a user