From 215ece4980fa37e0e4299eb505f91741334f3bb8 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:11:44 +0800 Subject: [PATCH 01/38] fix(html2md): retain complex tables and referenced fragments --- moli-html2md/README.md | 27 +- moli-html2md/src/anchors.rs | 178 ++++++++++++ moli-html2md/src/copy_tests.rs | 2 +- moli-html2md/src/dom.rs | 19 ++ moli-html2md/src/html_table.rs | 328 +++++++++++++++++++++ moli-html2md/src/lib.rs | 6 +- moli-html2md/src/machine.rs | 191 ++++++++++++- moli-html2md/src/table.rs | 36 +-- moli-html2md/tests/conversion.rs | 28 +- moli-html2md/tests/inline_boundaries.rs | 1 - moli-html2md/tests/regressions.rs | 360 +++++++++++++++++++++++- moli-html2md/tests/tables.rs | 130 +++++---- moli-html2md/tests/turndown/README.md | 5 +- 13 files changed, 1194 insertions(+), 117 deletions(-) create mode 100644 moli-html2md/src/anchors.rs create mode 100644 moli-html2md/src/html_table.rs diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 58209068c4e..434ef197392 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -70,18 +70,22 @@ missing trailing cells in short rows; the converter does not allocate a padded g Tables with `role="presentation"` or `role="none"` retain ordinary block expansion, in DOM order with blank lines between cells, even when structurally simple. Role -matching ignores ASCII case and surrounding HTML whitespace. The same fallback -applies to complex tables: non-unit spans, multiple `thead` rows, rows wider than +matching ignores ASCII case and surrounding HTML whitespace. Complex tables use +content-only HTML: non-unit spans, multiple `thead` rows, rows wider than an explicit header, or cells containing nested tables, lists, headings, quotes -or preformatted blocks. Inner simple tables can still produce GFM when their -containing layout table is expanded. Other roles, borders and CSS do not select -another conversion path. Complex layouts remain readable blocks instead of raw -HTML tables. +or preformatted blocks. This retains header, cell and nested-table associations +that Markdown pipe tables cannot express. Inner simple tables can still produce +GFM when an explicitly presentational containing table is expanded. Scripts, +styles and event handlers are omitted from HTML table output. Supported output includes headings, paragraphs, emphasis, strikethrough, links, -images, lists, blockquotes, hard breaks, media source links, readable form values, -separated form choices, and fenced code. Inline HTML is used when +images, lists, blockquotes, hard breaks, media source links, separated form choices, +and fenced code. Inline HTML is used when Markdown delimiters cannot express an emphasis boundary. +Superscripts and subscripts retain their HTML markers. Complete author-supplied +TeX spans retain their commands and indices. Native MathML retains mathematical +markup and presentation attributes, with alternate annotations omitted, so +fractions, roots, matrices and powers do not collapse into ambiguous plain text. Script/style/head/noscript/template subtrees are omitted. `max_depth` counts the supplied root as depth zero; nodes at or beyond the limit @@ -94,8 +98,11 @@ spaces and blank lines. Empty blocks still separate surrounding text. This is a structural content dump. It omits non-rendered serialized state blobs but retains content in collapsed panels and inactive tabs. It does not evaluate -stylesheets, layout, or JavaScript, and cannot preserve all HTML presentation (for example, arbitrary -ordered-list numbering or table spanning). DOM snapshots used by Moli's optional +stylesheets, layout, or JavaScript. Callers may supply computed block boundaries +through the optional DOM query. Moli's page exporter additionally filters current +CSS-hidden UI while retaining explicitly linked disclosure content. The converter +cannot preserve all HTML presentation (for example, arbitrary ordered-list numbering). +DOM snapshots used by Moli's optional strip/base/frame transformations remain the responsibility of the caller. ## Verification diff --git a/moli-html2md/src/anchors.rs b/moli-html2md/src/anchors.rs new file mode 100644 index 00000000000..a4d8145a758 --- /dev/null +++ b/moli-html2md/src/anchors.rs @@ -0,0 +1,178 @@ +use std::collections::HashSet; + +use crate::{Dom, NodeKind}; + +pub(crate) struct Targets { + referenced: HashSet, + ids: HashSet, +} + +impl Targets { + pub(crate) fn is_empty(&self) -> bool { + self.referenced.is_empty() + } + + pub(crate) fn contains(&self, value: &str) -> bool { + self.referenced.contains(value) + } + + pub(crate) fn targets<'a, D: Dom + ?Sized>(&self, dom: &'a D, node: D::NodeId) -> Vec<&'a str> { + let mut targets = Vec::with_capacity(2); + if let Some(id) = dom.attribute(node, "id").filter(|id| self.contains(id)) { + targets.push(id); + } + if dom.node_kind(node) == NodeKind::Element("a") + && let Some(name) = dom + .attribute(node, "name") + .filter(|name| self.contains(name) && !self.ids.contains(*name)) + && !targets.contains(&name) + { + targets.push(name); + } + targets + } +} + +pub(crate) fn referenced(dom: &D, root: D::NodeId, limit: usize) -> Targets { + let mut available = HashSet::new(); + let mut element_ids = HashSet::new(); + if !dom.may_have_fragment_links() { + return Targets { + referenced: available, + ids: element_ids, + }; + } + walk(dom, root, limit, |node| { + for attribute in ["id", "name"] { + if attribute == "name" && dom.node_kind(node) != NodeKind::Element("a") { + continue; + } + if let Some(value) = dom.attribute(node, attribute) { + available.insert(value.to_owned()); + if attribute == "id" { + element_ids.insert(value.to_owned()); + } + } + } + false + }); + let mut referenced = HashSet::new(); + walk(dom, root, limit, |node| { + if let Some(fragment) = dom + .attribute(node, "href") + .and_then(|href| href.strip_prefix('#')) + && !fragment.is_empty() + { + // HTML fragment navigation tries the literal fragment first and + // only percent-decodes it when no literal target exists. + if available.contains(fragment) { + referenced.insert(fragment.to_owned()); + } else { + let decoded = decode_fragment(fragment); + if available.contains(&decoded) { + referenced.insert(decoded); + } + } + } + false + }); + Targets { + referenced, + ids: element_ids, + } +} + +pub(crate) fn contains( + dom: &D, + root: D::NodeId, + limit: usize, + ids: &Targets, +) -> bool { + walk(dom, root, limit, |node| !ids.targets(dom, node).is_empty()) +} + +pub(crate) fn within( + dom: &D, + root: D::NodeId, + limit: usize, + ids: &Targets, +) -> Vec { + let mut found = Vec::new(); + walk(dom, root, limit, |node| { + for id in ids.targets(dom, node) { + if !found.iter().any(|existing| existing == id) { + found.push(id.to_owned()); + } + } + false + }); + found +} + +fn walk( + dom: &D, + root: D::NodeId, + limit: usize, + mut found: impl FnMut(D::NodeId) -> bool, +) -> bool { + let mut stack = vec![(Some(root), 0, false)]; + while let Some((node, depth, siblings)) = stack.pop() { + let Some(node) = node else { + continue; + }; + if siblings { + stack.push((dom.next_sibling(node), depth, true)); + } + if depth >= limit { + continue; + } + if matches!( + dom.node_kind(node), + NodeKind::Other + | NodeKind::Element("head" | "script" | "style" | "noscript" | "template") + ) { + continue; + } + if found(node) { + return true; + } + stack.push((dom.first_child(node), depth + 1, true)); + } + false +} + +pub(crate) fn decode_fragment(fragment: &str) -> String { + let bytes = fragment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut offset = 0; + while offset < bytes.len() { + if bytes[offset] == b'%' && offset + 2 < bytes.len() { + let hex = |byte: u8| (byte as char).to_digit(16); + if let (Some(high), Some(low)) = (hex(bytes[offset + 1]), hex(bytes[offset + 2])) { + decoded.push((high * 16 + low) as u8); + offset += 3; + continue; + } + } + decoded.push(bytes[offset]); + offset += 1; + } + String::from_utf8(decoded).unwrap_or_else(|_| fragment.to_owned()) +} + +pub(crate) fn markup(id: &str) -> String { + let mut result = String::from(" result.push_str("&"), + '"' => result.push_str("""), + '<' => result.push_str("<"), + '>' => result.push_str(">"), + '\n' => result.push_str(" "), + '\r' => result.push_str(" "), + character => result.push(character), + } + } + result.push_str("\">"); + result +} diff --git a/moli-html2md/src/copy_tests.rs b/moli-html2md/src/copy_tests.rs index 9439eecea1a..123fabb65dd 100644 --- a/moli-html2md/src/copy_tests.rs +++ b/moli-html2md/src/copy_tests.rs @@ -28,7 +28,7 @@ fn nested_table_fallback_copies_text_linearly() { assert_eq!(actual.matches(&after).count(), depth); assert_eq!( support::rendered_html(&actual).matches("").count(), - 1 + depth ); // Count actual bytes copied when materializing fragments; completed // blocks move into parents without copying their text. This measures diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs index 15a86903a37..e13c643f97e 100644 --- a/moli-html2md/src/dom.rs +++ b/moli-html2md/src/dom.rs @@ -32,4 +32,23 @@ pub trait Dom { fn control_value(&self, _node: Self::NodeId) -> Option<&str> { None } + + /// A caller with computed CSS can retain block boundaries introduced by + /// stylesheets (for example, links rendered as separate action buttons). + /// Structural-only adapters can leave this at its default. + fn has_block_layout(&self, _node: Self::NodeId) -> bool { + false + } + + /// A caller with computed CSS can retain boundaries between independent + /// inline boxes such as tabs, badges, and compact action controls. + fn has_text_boundary(&self, _node: Self::NodeId) -> bool { + false + } + + /// Adapters that already inventory elements can skip the anchor prepass + /// when the document contains no local-fragment links. + fn may_have_fragment_links(&self) -> bool { + true + } } diff --git a/moli-html2md/src/html_table.rs b/moli-html2md/src/html_table.rs new file mode 100644 index 00000000000..1dbfec6bd8f --- /dev/null +++ b/moli-html2md/src/html_table.rs @@ -0,0 +1,328 @@ +use std::collections::HashSet; + +use crate::{Dom, NodeKind}; + +/// Markdown has no merged cells, nested tables, or block content inside cells. +/// Detect those structures before conversion so a table is traversed at most +/// twice, even when it contains thousands of nested tables. +pub(crate) fn needed( + dom: &D, + root: D::NodeId, + depth: usize, + limit: usize, +) -> bool { + let mut stack = vec![(dom.first_child(root), depth + 1)]; + while let Some((node, depth)) = stack.pop() { + let Some(node) = node else { + continue; + }; + if depth >= limit { + continue; + } + stack.push((dom.next_sibling(node), depth)); + if let NodeKind::Element(tag) = dom.node_kind(node) { + if matches!( + tag, + "table" + | "pre" + | "blockquote" + | "ul" + | "ol" + | "li" + | "hr" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + ) { + return true; + } + if matches!(tag, "td" | "th") + && ["rowspan", "colspan"].iter().any(|attr| { + dom.attribute(node, attr) + .is_some_and(|value| value.trim().parse::() != Ok(1)) + }) + { + return true; + } + } + stack.push((dom.first_child(node), depth + 1)); + } + false +} + +pub(crate) fn has_cells( + dom: &D, + root: D::NodeId, + depth: usize, + limit: usize, +) -> bool { + let mut stack = vec![(dom.first_child(root), depth + 1)]; + while let Some((node, depth)) = stack.pop() { + let Some(node) = node else { + continue; + }; + if depth >= limit { + continue; + } + if matches!(dom.node_kind(node), NodeKind::Element("th" | "td")) { + return true; + } + stack.push((dom.next_sibling(node), depth)); + stack.push((dom.first_child(node), depth + 1)); + } + false +} + +enum Task { + Node(Id, usize), + Siblings(Option, usize), + Close(String), + Literal(&'static str), +} + +/// A compact HTML block is valid Markdown and preserves the original cell +/// associations. Keep content markup only; scripts, styles, event handlers +/// and unrelated presentation attributes are never copied. +pub(crate) fn render( + dom: &D, + root: D::NodeId, + depth: usize, + limit: usize, + anchor_targets: &crate::anchors::Targets, +) -> String { + let mut output = String::new(); + let mut emitted_anchors = HashSet::new(); + let mut tasks = vec![Task::Node(root, depth)]; + while let Some(task) = tasks.pop() { + match task { + Task::Literal(value) => output.push_str(value), + Task::Close(tag) => { + output.push_str("'); + } + Task::Siblings(None, _) => {} + Task::Siblings(Some(node), depth) => { + tasks.push(Task::Siblings(dom.next_sibling(node), depth)); + tasks.push(Task::Node(node, depth)); + } + Task::Node(node, depth) => { + if depth >= limit { + continue; + } + if matches!(dom.node_kind(node), NodeKind::Element(_)) { + if dom.has_block_layout(node) { + output.push_str("
"); + tasks.push(Task::Close("div".to_owned())); + } else if dom.has_text_boundary(node) { + output.push(' '); + tasks.push(Task::Literal(" ")); + } + } + if let NodeKind::Element(tag) = dom.node_kind(node) + && !allowed(tag) + { + for id in anchor_targets.targets(dom, node) { + if emitted_anchors.insert(id.to_owned()) { + output.push_str(&crate::anchors::markup(id)); + } + } + } + match dom.node_kind(node) { + NodeKind::Text(text) => escape(text, &mut output), + NodeKind::Element( + "head" | "title" | "script" | "style" | "noscript" | "template", + ) + | NodeKind::Other => {} + NodeKind::Element("math") => { + for id in crate::anchors::within(dom, node, limit - depth, anchor_targets) { + if emitted_anchors.insert(id.clone()) { + output.push_str(&crate::anchors::markup(&id)); + } + } + output.push_str(&crate::mathml::render(dom, node, limit - depth)) + } + NodeKind::Element("input") => { + if let Some(value) = crate::form::input_text(dom, node) { + escape(&value, &mut output); + } + } + NodeKind::Element("textarea") => { + if let Some(value) = crate::form::textarea_text(dom, node) { + escape(&value, &mut output); + } else { + tasks.push(Task::Siblings(dom.first_child(node), depth + 1)); + } + } + NodeKind::Element(tag) => { + if crate::visibility::nonrendered_serialized_state(dom, node) { + continue; + } + if matches!(tag, "iframe" | "video" | "audio") { + let urls = crate::media::sources(dom, node, tag, depth, limit); + if !urls.is_empty() { + output.push_str("
"); + if tag == "video" + && let Some(poster) = dom + .attribute(node, "poster") + .filter(|src| crate::media::safe_url(src, true)) + { + output.push_str("\"\""); + } + for url in urls + .into_iter() + .filter(|url| crate::media::safe_url(url, false)) + { + output.push_str(""); + escape(crate::media::label(dom, node, tag), &mut output); + output.push_str("
"); + } + output.push_str("
"); + continue; + } + } + let retained = if allowed(tag) { Some(tag) } else { None }; + if let Some(tag) = retained { + output.push('<'); + output.push_str(tag); + for name in [ + "rowspan", "colspan", "scope", "headers", "id", "href", "src", + "alt", "title", "start", "reversed", "value", "label", "selected", + "multiple", "disabled", "name", + ] { + let image_source = if name == "src" && tag == "img" { + crate::media::source(dom, node) + } else { + None + }; + let value = if name == "src" && tag == "img" { + image_source.as_deref() + } else { + dom.attribute(node, name) + }; + if let Some(value) = value { + if matches!(name, "href" | "src") + && !crate::media::safe_url(value, name == "src") + { + continue; + } + output.push(' '); + output.push_str(name); + output.push_str("=\""); + escape(value, &mut output); + output.push('"'); + } + } + output.push('>'); + if !matches!(tag, "br" | "hr" | "img") { + tasks.push(Task::Close(tag.to_owned())); + } + } + if let Some((math, relative_depth)) = + crate::mathml::primary_alternative(dom, node, limit - depth) + { + for id in + crate::anchors::within(dom, node, limit - depth, anchor_targets) + { + if emitted_anchors.insert(id.clone()) { + output.push_str(&crate::anchors::markup(&id)); + } + } + output.push_str(&crate::mathml::render( + dom, + math, + limit - depth - relative_depth, + )); + } else { + if let Some(label) = crate::content::fallback_text(dom, node) + && !crate::content::has_readable_content(dom, node, limit - depth) + { + escape(&label, &mut output); + } else { + tasks.push(Task::Siblings(dom.first_child(node), depth + 1)); + } + } + } + NodeKind::Document => { + tasks.push(Task::Siblings(dom.first_child(node), depth + 1)) + } + } + } + } + } + output +} + +fn allowed(tag: &str) -> bool { + matches!( + tag, + "table" + | "thead" + | "tbody" + | "tfoot" + | "tr" + | "td" + | "th" + | "caption" + | "select" + | "optgroup" + | "option" + | "button" + | "textarea" + | "p" + | "div" + | "span" + | "a" + | "img" + | "br" + | "hr" + | "strong" + | "b" + | "em" + | "i" + | "s" + | "del" + | "sup" + | "sub" + | "code" + | "pre" + | "blockquote" + | "ul" + | "ol" + | "li" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + | "dl" + | "dt" + | "dd" + | "details" + | "summary" + ) +} + +fn escape(text: &str, output: &mut String) { + for ch in text.chars() { + match ch { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '"' => output.push_str("""), + // Raw HTML blocks end at an empty Markdown line. Entities preserve + // preformatted line endings without prematurely ending the block. + '\n' => output.push_str(" "), + '\r' => output.push_str(" "), + _ => output.push(ch), + } + } +} diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs index ec7a16b8cd5..8bbec691b5d 100644 --- a/moli-html2md/src/lib.rs +++ b/moli-html2md/src/lib.rs @@ -1,7 +1,9 @@ +mod anchors; mod content; mod converter; mod dom; mod form; +mod html_table; mod machine; mod math; mod mathml; @@ -16,10 +18,10 @@ pub use converter::{Converter, convert}; pub use dom::{Dom, NodeKind}; pub use options::Options; +pub use media::{SrcsetCandidate, parse_srcset}; + #[cfg(test)] extern crate self as moli_html2md; #[cfg(test)] mod copy_tests; - -pub use media::{SrcsetCandidate, parse_srcset}; diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 9351431ba17..ac4ca227668 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -3,6 +3,7 @@ use crate::table::Table; use crate::visibility::nonrendered_serialized_state; use crate::writer::{Style, Writer, longest_run}; use crate::{Dom, NodeKind, Options}; +use std::collections::HashSet; enum Task<'a, Id> { Visit(Id, usize), @@ -41,6 +42,8 @@ struct Machine<'a, D: Dom + ?Sized> { tables: Vec>, raw: String, serial: usize, + anchor_targets: crate::anchors::Targets, + emitted_anchors: HashSet, } pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Options) -> String { @@ -53,6 +56,8 @@ pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Optio tables: Vec::new(), raw: String::new(), serial: 0, + anchor_targets: crate::anchors::referenced(dom, root, options.max_depth), + emitted_anchors: HashSet::new(), }; machine.run(); machine.take_writer().into_string() @@ -210,6 +215,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if depth >= self.options.max_depth { return; } + if self.dom.has_block_layout(node) { + self.writer().boundary(2); + self.tasks.push(Task::Boundary); + } else if self.dom.has_text_boundary(node) { + self.writer().boundary(1); + self.tasks.push(Task::ChoiceBoundary); + } let tag = match self.dom.node_kind(node) { NodeKind::Document => { self.children(node, depth + 1); @@ -225,11 +237,21 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if nonrendered_serialized_state(self.dom, node) { return; } - if let Some(table) = self.tables.last_mut() { - table.visit_element(tag); + if tag != "table" + && !matches!( + tag, + "head" | "script" | "style" | "noscript" | "template" | "title" + ) + { + for id in self.anchor_targets.targets(self.dom, node) { + if self.emitted_anchors.insert(id.to_owned()) { + self.writer().inline_html(&crate::anchors::markup(id)); + } + } } match tag { "math" => { + self.emit_anchors_within(node, self.options.max_depth - depth); let markup = crate::mathml::render(self.dom, node, self.options.max_depth - depth); self.writer().inline_html(&markup); return; @@ -313,6 +335,39 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } "iframe" | "video" | "audio" if self.media(node, tag, depth) => return, "code" | "pre" => { + let remaining = self.options.max_depth - depth; + if subtree_has_link(self.dom, node, remaining) + || (!self.anchor_targets.is_empty() + && crate::anchors::contains( + self.dom, + node, + remaining, + &self.anchor_targets, + )) + { + let html = crate::html_table::render( + self.dom, + node, + depth, + self.options.max_depth, + &self.anchor_targets, + ); + if tag == "pre" { + self.writer().block(html.into(), 2, 2); + } else { + self.writer().inline_html(&html); + } + return; + } + if tag == "pre" && !self.anchor_targets.is_empty() { + for id in + crate::anchors::within(self.dom, node, remaining, &self.anchor_targets) + { + if self.emitted_anchors.insert(id.clone()) { + self.writer().inline_html(&crate::anchors::markup(&id)); + } + } + } self.raw.clear(); let end = if tag == "pre" { Task::EndPre(self.language(node)) @@ -330,6 +385,24 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { return; } "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => { + let remaining = self.options.max_depth - depth; + if subtree_has_nested_heading(self.dom, node, remaining) { + self.writer().boundary(2); + self.tasks.push(Task::Boundary); + self.children(node, depth + 1); + return; + } + if subtree_has_block(self.dom, node, remaining) { + let html = crate::html_table::render( + self.dom, + node, + depth, + self.options.max_depth, + &self.anchor_targets, + ); + self.writer().block(html.into(), 2, 2); + return; + } self.capture(); self.writer().heading(); self.tasks @@ -342,6 +415,32 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { "ul" | "ol" => self.start_list(node, tag == "ol"), "li" => self.start_item(), "table" => { + let presentation = self.dom.attribute(node, "role").is_some_and(|role| { + matches!( + role.trim().to_ascii_lowercase().as_str(), + "presentation" | "none" + ) + }); + if (!presentation + && crate::html_table::needed(self.dom, node, depth, self.options.max_depth)) + || (!self.anchor_targets.is_empty() + && crate::anchors::contains( + self.dom, + node, + self.options.max_depth - depth, + &self.anchor_targets, + )) + { + let html = crate::html_table::render( + self.dom, + node, + depth, + self.options.max_depth, + &self.anchor_targets, + ); + self.writer().block(html.into(), 2, 2); + return; + } if let Some(mut table) = Table::from_dom(self.dom, node, depth, self.options.max_depth) { @@ -353,6 +452,19 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } return; } + if !presentation + && crate::html_table::has_cells(self.dom, node, depth, self.options.max_depth) + { + let html = crate::html_table::render( + self.dom, + node, + depth, + self.options.max_depth, + &self.anchor_targets, + ); + self.writer().block(html.into(), 2, 2); + return; + } self.writer().boundary(2); self.tasks.push(Task::Boundary); } @@ -365,6 +477,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if crate::mathml::primary_alternative(self.dom, node, self.options.max_depth - depth) .is_some() { + self.emit_anchors_within(node, self.options.max_depth - depth); self.tasks .push(Task::MathChildren(self.dom.first_child(node), depth + 1)); return; @@ -378,6 +491,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.children(node, depth + 1); } + fn emit_anchors_within(&mut self, node: D::NodeId, remaining: usize) { + for id in crate::anchors::within(self.dom, node, remaining, &self.anchor_targets) { + if self.emitted_anchors.insert(id.clone()) { + self.writer().inline_html(&crate::anchors::markup(&id)); + } + } + } + fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); let poster = (tag == "video") @@ -585,6 +706,72 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } +fn subtree_has_nested_heading(dom: &D, root: D::NodeId, limit: usize) -> bool { + let mut pending = Vec::new(); + let mut child = dom.first_child(root); + while let Some(id) = child { + pending.push((id, 1)); + child = dom.next_sibling(id); + } + while let Some((node, depth)) = pending.pop() { + if matches!( + dom.node_kind(node), + NodeKind::Element("h1" | "h2" | "h3" | "h4" | "h5" | "h6") + ) { + return true; + } + if depth + 1 >= limit { + continue; + } + let mut child = dom.first_child(node); + while let Some(id) = child { + pending.push((id, depth + 1)); + child = dom.next_sibling(id); + } + } + false +} + +fn subtree_has_link(dom: &D, root: D::NodeId, limit: usize) -> bool { + let mut pending = vec![(dom.first_child(root), 1)]; + while let Some((node, depth)) = pending.pop() { + let Some(node) = node else { + continue; + }; + if depth >= limit { + continue; + } + pending.push((dom.next_sibling(node), depth)); + if matches!(dom.node_kind(node), NodeKind::Element("a")) + && dom + .attribute(node, "href") + .is_some_and(|href| !href.is_empty() && crate::media::safe_url(href, false)) + { + return true; + } + pending.push((dom.first_child(node), depth + 1)); + } + false +} + +fn subtree_has_block(dom: &D, root: D::NodeId, limit: usize) -> bool { + let mut pending = vec![(dom.first_child(root), 1)]; + while let Some((node, depth)) = pending.pop() { + let Some(node) = node else { + continue; + }; + if depth >= limit { + continue; + } + pending.push((dom.next_sibling(node), depth)); + if matches!(dom.node_kind(node), NodeKind::Element(tag) if is_structural_block(tag)) { + return true; + } + pending.push((dom.first_child(node), depth + 1)); + } + false +} + fn class_language(class: Option<&str>) -> Option<&str> { class? .split_ascii_whitespace() diff --git a/moli-html2md/src/table.rs b/moli-html2md/src/table.rs index f29e72208ea..6a4ee474053 100644 --- a/moli-html2md/src/table.rs +++ b/moli-html2md/src/table.rs @@ -10,7 +10,6 @@ pub(crate) struct Table { cells: std::vec::IntoIter>, content: Vec, in_cell: bool, - has_complex_content: bool, rows: Vec, separators: Vec<&'static str>, has_header: bool, @@ -37,7 +36,6 @@ impl Table { cells: Vec::new().into_iter(), content: Vec::new(), in_cell: false, - has_complex_content: false, rows: Vec::new(), separators: Vec::new(), has_header: false, @@ -166,39 +164,9 @@ impl Table { self.in_cell = false; } - pub(crate) fn visit_element(&mut self, tag: &str) { - if self.in_cell - && matches!( - tag, - "table" - | "pre" - | "blockquote" - | "ul" - | "ol" - | "li" - | "hr" - | "h1" - | "h2" - | "h3" - | "h4" - | "h5" - | "h6" - ) - { - self.has_complex_content = true; - } - } - pub(crate) fn finish(self) -> Output { - if self.has_complex_content { - let mut output = Output::default(); - for cell in self.content.into_iter().filter(|cell| !cell.is_empty()) { - if !output.is_empty() { - output.push_text("\n\n".into()); - } - output.append(cell); - } - return output; + if self.content.iter().all(Output::is_empty) { + return Output::default(); } let mut output = String::new(); if !self.has_header { diff --git a/moli-html2md/tests/conversion.rs b/moli-html2md/tests/conversion.rs index 13d0740c0f6..2238d5d5b06 100644 --- a/moli-html2md/tests/conversion.rs +++ b/moli-html2md/tests/conversion.rs @@ -1,4 +1,5 @@ use moli_html2md::{Converter, Dom, NodeKind, Options, convert}; +mod support; // A second DOM implementation, independent of NativeDom or an HTML parser. // Its stable arena IDs and borrowed strings are enough for the converter. @@ -304,7 +305,9 @@ fn table_spans_do_not_duplicate_cell_text() { let row = dom.element(table, "tr"); let cell = dom.leaf(row, "td", "end"); dom.attr(cell, "colspan", "999999999999"); - assert_eq!(convert(&dom, 0), "one\n\ntwo\n\nend"); + let result = convert(&dom, 0); + assert!(result.contains("rowspan=\"2\">one
")); + assert!(result.contains("colspan=\"999999999999\">end")); } #[test] @@ -381,7 +384,9 @@ fn converts_deep_tables_on_a_small_thread_stack() { max_depth: usize::MAX, ..Options::default() }); - assert_eq!(converter.convert(&dom, 0), "| |\n| --- |\n| deep |"); + let result = converter.convert(&dom, 0); + assert_eq!(result.matches("
two
").count(), 10_000); + assert_eq!(result.matches("deep").count(), 1); }) .expect("spawn small-stack test") .join() @@ -409,8 +414,8 @@ fn converts_nested_header_tables_on_a_small_thread_stack() { }); let actual = converter.convert(&dom, 0); assert_eq!(actual.matches("Header").count(), 1_000); - assert_eq!(actual.matches("| --- |").count(), 1); - assert!(actual.ends_with("| deep |")); + assert_eq!(actual.matches("
").count(), 1_000); + assert_eq!(actual.matches("deep").count(), 1); }) .expect("spawn small-stack test") .join() @@ -508,10 +513,15 @@ fn markdown_parser_recovers_pipes_and_preformatted_code_inside_tables() { let cell = dom.element(row, "td"); dom.leaf(cell, "pre", " |b\nnext\n"); let markdown = convert(&dom, 0); - let html = rendered_html(&markdown); - assert_eq!( - html, - "

Code

\n

a|b

\n
  <a>|b\nnext\n
\n", - "{markdown}" + let html = support::rendered_html(&markdown); + assert!(html.contains("a|b"), "{html}"); + assert_eq!(html.matches(""), "{html}"); + assert!( + html.contains(""), + "{html}" + ); + assert_eq!(html.matches(">Group<").count(), 1); +} + +#[test] +fn nested_tables_stay_with_their_column_labels() { + let source = "
").count(), 2); + let parsed = support::Tree::parse(&html); + assert!( + parsed + .nodes + .iter() + .any(|node| node.text.as_deref() == Some(" |b\nnext\n")), + "{html}" ); } diff --git a/moli-html2md/tests/inline_boundaries.rs b/moli-html2md/tests/inline_boundaries.rs index 43187839fda..bc63c9b1db2 100644 --- a/moli-html2md/tests/inline_boundaries.rs +++ b/moli-html2md/tests/inline_boundaries.rs @@ -20,7 +20,6 @@ fn inline_code_breaks_separate_words_without_formatting_markers() { for html in [ "git
status", "git
status", - "git
status", ] { for preformatted_code in [false, true] { let result = markdown_with( diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index 98837d5961e..b36261970ae 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -1,6 +1,7 @@ mod support; -use moli_html2md::{Converter, Options, parse_srcset}; +use moli_html2md::{Converter, Dom, NodeKind, Options, parse_srcset}; +use std::cell::Cell; use support::{Tree, rendered_html}; fn markdown(html: &str, preformatted_code: bool) -> String { @@ -846,10 +847,7 @@ fn lazy_media_uses_declared_resources_when_standard_sources_are_unavailable() { "
Photo
", false, ); - assert!( - rendered_html(&table).contains("src=\"/actual.jpg\""), - "{table}" - ); + assert!(table.contains("src=\"/actual.jpg\""), "{table}"); assert!(!table.contains("spacer.gif"), "{table}"); } @@ -962,6 +960,7 @@ fn math_streams_across_transparent_wrappers_in_every_inline_container() { r"Equation

$x + y$

$z + 1$

", false, ); + assert!(result.contains(""), "{result}"); assert!(result.contains("$x + y$\n\n$z + 1$"), "{result}"); } @@ -1047,3 +1046,354 @@ fn code_excludes_embedded_copy_controls_and_scripts() { assert!(!result.contains("Copy code"), "{result}"); assert!(!result.contains("advert"), "{result}"); } + +#[test] +fn empty_layout_tables_do_not_invent_data() { + let result = markdown( + "

Article

Video unavailable
MetricValue
Rate
", + false, + ); + assert!(result.contains("Video unavailable"), "{result}"); + assert!(!result.contains("| |\n| --- |\n| |"), "{result}"); + assert!(result.contains("| Rate | |"), "{result}"); +} + +#[test] +fn merged_cells_keep_calendar_and_group_associations() { + let source = "
MonTueWed
1
GroupA
B
"; + let result = markdown(source, false); + let html = rendered_html(&result); + assert!(html.contains("		1GroupA
TodayMonth
Alice20
Bob100
"; + let html = rendered_html(&markdown(source, false)); + assert!(html.contains("TodayMonth"), "{html}"); + assert!(html.contains(""), "{html}"); + assert!(html.contains(""), "{html}"); + assert!(html.contains(""), "{html}"); +} + +#[test] +fn complex_tables_keep_media_choices_and_literal_content_inert() { + let source = "
Alice20Bob100
<script>literal</script>\n\nline
Bad link
"; + let html = rendered_html(&markdown(source, false)); + assert!(html.contains("href=\"/lecture\">Lecture"), "{html}"); + assert!( + html.contains(""), + "{html}" + ); + assert!( + html.contains("<script>literal</script>"), + "{html}" + ); + assert!(!html.contains("onclick"), "{html}"); + assert!(!html.contains("