diff --git a/CHANGELOG.md b/CHANGELOG.md index 620b9cc..7907abb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `extractDocument` / `extractDocumentSync` return Markdown with title, format, warnings, stats and page metadata. - `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats. +- Main-content detection: `MarkkitBuilder::main_content(true)` (CLI `--main-content`) keeps only + the article body of a web page, Readability-style, dropping navigation, sidebars, comments, share + bars and related-link blocks. Off by default; include selectors take precedence; falls back to the + full page when no article-like block is found. ### Security diff --git a/README.md b/README.md index b39c035..78b51c5 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,9 @@ markkit extract https://example.com --include "article" --exclude "nav,footer" # JSON output with format, title and page metadata markkit extract https://example.com --json +# Keep only the article body (drops sidebars, comments, share bars) +markkit extract https://example.com/post --main-content + # Resolve relative links in a saved page markkit extract --file page.html --base-url https://example.com/blog/ diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 9541120..36abb6d 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -50,6 +50,10 @@ enum Commands { /// (URLs use the final response URL) #[arg(long)] base_url: Option, + + /// Keep only the detected main content (article body) of HTML pages + #[arg(long)] + main_content: bool, }, } @@ -67,6 +71,7 @@ async fn main() -> Result<()> { max_size, json: output_json, base_url, + main_content, } => { let read_stdin = stdin || source.as_deref() == Some("-"); @@ -74,7 +79,9 @@ async fn main() -> Result<()> { anyhow::bail!("source is required unless --stdin or \"-\" is used"); } - let mut builder = Markkit::builder().max_input_size(max_size); + let mut builder = Markkit::builder() + .max_input_size(max_size) + .main_content(main_content); for sel in &include_selectors { builder = builder.include_selector(sel); diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index 0f3eb3b..cf05083 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -50,6 +50,10 @@ pub struct MarkkitBuilder { pub track_sources: bool, /// IR passes run after parsing, before rendering (see [`crate::transform`]). pub transforms: crate::transform::TransformChain, + /// Keep only the detected main content (article body) of HTML pages, + /// Readability-style. Include selectors win; falls back to the full body when + /// nothing scores. Default: false. + pub main_content: bool, } impl Default for MarkkitBuilder { @@ -82,6 +86,7 @@ impl Default for MarkkitBuilder { base_url: None, track_sources: false, transforms: Default::default(), + main_content: false, } } } @@ -189,6 +194,13 @@ impl MarkkitBuilder { self } + /// Extract only the detected main content of HTML pages (drops sidebars, + /// comments, share bars and other boilerplate). + pub fn main_content(mut self, yes: bool) -> Self { + self.main_content = yes; + self + } + pub fn build(self) -> Markkit { Markkit::new(self) } diff --git a/crates/markkit/src/html/mod.rs b/crates/markkit/src/html/mod.rs index 4755900..945aebc 100644 --- a/crates/markkit/src/html/mod.rs +++ b/crates/markkit/src/html/mod.rs @@ -4,6 +4,7 @@ mod metadata; mod parser; +mod readability; mod urls; pub use parser::HtmlParser; diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 01c24ff..622ba9c 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -19,7 +19,7 @@ //! - `
` becomes `Figure` only when it has a non-empty `
`; //! otherwise it stays a transparent container as before. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::collections::HashMap; use ego_tree::{NodeId, NodeRef}; @@ -32,7 +32,7 @@ use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, Tab use url::Url; use super::metadata::PageInfo; -use super::urls; +use super::{readability, urls}; /// Maximum recursion depth for nested elements. const MAX_DEPTH: usize = 200; @@ -49,6 +49,8 @@ pub struct HtmlParser<'a> { base: Option, /// Cached 1-based index of an element among same-name element siblings. sibling_index: RefCell>, + /// Main-content mode: also drop "unlikely" subtrees (sidebars, share bars, ...). + drop_unlikely: Cell, } impl<'a> HtmlParser<'a> { @@ -58,6 +60,7 @@ impl<'a> HtmlParser<'a> { footnote_labels: RefCell::new(HashMap::new()), base: urls::parse_base(config.base_url.as_deref()), sibling_index: RefCell::new(HashMap::new()), + drop_unlikely: Cell::new(false), } } @@ -87,27 +90,13 @@ impl<'a> HtmlParser<'a> { footnote_labels: RefCell::new(HashMap::new()), sibling_index: RefCell::new(HashMap::new()), base: urls::document_base(dom, self.base.clone()), + drop_unlikely: Cell::new(false), } } fn convert_dom(&self, dom: &Html) -> Result { let root = dom.root_element(); - let content_nodes: Vec> = - if !self.config.include_selectors.is_empty() { - let sels: Vec = self - .config - .include_selectors - .iter() - .filter_map(|s| Selector::parse(s).ok()) - .collect(); - sels.iter().flat_map(|sel| dom.select(sel)).collect() - } else { - let body_sel = Selector::parse("body").unwrap(); - let body: Vec<_> = dom.select(&body_sel).collect(); - if body.is_empty() { vec![root] } else { body } - }; - let exclude_selectors: Vec = self .config .exclude_selectors @@ -116,10 +105,40 @@ impl<'a> HtmlParser<'a> { .collect(); let mut nodes = Vec::new(); - for content_root in content_nodes { - self.convert_children(content_root, &exclude_selectors, &mut nodes, 0); + if !self.config.include_selectors.is_empty() { + let sels: Vec = self + .config + .include_selectors + .iter() + .filter_map(|s| Selector::parse(s).ok()) + .collect(); + for content_root in sels.iter().flat_map(|sel| dom.select(sel)) { + self.convert_children(content_root, &exclude_selectors, &mut nodes, 0); + } + return Ok(Document::new(normalize_blocks(nodes))); + } + + let body_sel = Selector::parse("body").unwrap(); + let body = dom.select(&body_sel).next().unwrap_or(root); + + if self.config.main_content + && let Some(roots) = readability::main_content(body, &exclude_selectors) + { + self.drop_unlikely.set(true); + for r in roots { + if r.whole && !is_transparent_container(r.el.value().name()) { + if let Some(mut node) = self.convert_element(r.el, &exclude_selectors, 1) { + self.mark_source(&mut node, r.el); + nodes.push(node); + } + } else { + self.convert_children(r.el, &exclude_selectors, &mut nodes, 0); + } + } + return Ok(Document::new(normalize_blocks(nodes))); } + self.convert_children(body, &exclude_selectors, &mut nodes, 0); Ok(Document::new(normalize_blocks(nodes))) } @@ -170,6 +189,7 @@ impl<'a> HtmlParser<'a> { fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool { exclude.iter().any(|sel| sel.matches(el)) + || (self.drop_unlikely.get() && readability::is_unlikely(*el)) } fn convert_children( diff --git a/crates/markkit/src/html/readability.rs b/crates/markkit/src/html/readability.rs new file mode 100644 index 0000000..0a6fa80 --- /dev/null +++ b/crates/markkit/src/html/readability.rs @@ -0,0 +1,427 @@ +//! Main-content detection: a Mozilla Readability-style scorer over the scraper DOM. +//! +//! Decisions: +//! - Opt-in (`MarkkitBuilder::main_content`), third strategy next to include selectors (which +//! win) and the full body. When no candidate carries enough text, extraction falls back to +//! the full body, so turning it on never yields an empty result for a page with content. +//! - Scoring follows Readability: paragraph-like blocks (`p`, `pre`, `td`, `blockquote`, +//! leaf `div`/`section`) with >= 25 chars score `1 + commas + min(chars / 100, 3)`, credited +//! to the parent (full), grandparent (1/2) and great-grandparent (1/3). Ancestors start +//! from a tag score plus a +-25 class/id weight. The final score is scaled by +//! `1 - link density`; siblings of the winner with a related score or prose-like text are kept. +//! - "Unlikely" subtrees (class/id/role such as `sidebar`, `comment`, `share`, `navigation`) +//! are skipped for scoring and also dropped from the rendered main content. +//! - Bounded work: one iterative traversal; paragraphs are disjoint (leaf blocks only), so +//! text measurement is linear. Link density is computed only for the top +//! [`MAX_FINALISTS`] candidates. No recursion, so DOM depth cannot overflow the stack. + +use std::collections::HashMap; + +use ego_tree::NodeId; +use ego_tree::iter::Edge; +use scraper::{ElementRef, Node, Selector}; + +/// Candidates whose link density is evaluated. +const MAX_FINALISTS: usize = 10; +/// Shortest paragraph text (chars) that scores. +const MIN_PARAGRAPH_CHARS: usize = 25; +/// Main content with less text than this falls back to the full body. +const MIN_CONTENT_CHARS: usize = 140; + +/// Content roots in document order. `whole` = render the element itself, otherwise its children. +pub(crate) struct ContentRoot<'a> { + pub el: ElementRef<'a>, + pub whole: bool, +} + +/// Pick the main-content roots under `body`, or `None` to fall back to the full body. +pub(crate) fn main_content<'a>( + body: ElementRef<'a>, + exclude: &[Selector], +) -> Option>> { + let mut scores: HashMap = HashMap::new(); + let mut order: Vec = Vec::new(); + + let mut skip: Option = None; + for edge in body.traverse() { + let node = match edge { + Edge::Open(n) => n, + Edge::Close(n) => { + if skip == Some(n.id()) { + skip = None; + } + continue; + } + }; + if skip.is_some() { + continue; + } + let Some(el) = ElementRef::wrap(node) else { + continue; + }; + if el.id() != body.id() && (is_unlikely(el) || exclude.iter().any(|s| s.matches(&el))) { + skip = Some(el.id()); + continue; + } + if !is_paragraph_like(el) { + continue; + } + let len = text_len(el); + if len < MIN_PARAGRAPH_CHARS { + continue; + } + let commas = el + .text() + .map(|t| t.matches([',', '，', '、']).count()) + .sum::(); + let score = 1.0 + commas as f64 + (len as f64 / 100.0).floor().min(3.0); + + let mut ancestor = el.parent().and_then(ElementRef::wrap); + for level in 0..3 { + let Some(a) = ancestor else { break }; + if a.id() == body.id() && level > 0 { + break; + } + let entry = scores.entry(a.id()).or_insert_with(|| { + order.push(a.id()); + initial_score(a) + }); + *entry += score / [1.0, 2.0, 3.0][level]; + if a.id() == body.id() { + break; + } + ancestor = a.parent().and_then(ElementRef::wrap); + } + } + + let tree = body.tree(); + let element = |id: NodeId| tree.get(id).and_then(ElementRef::wrap); + + // Finalists by raw score (stable: document order breaks ties), then link-density scaling. + let mut finalists: Vec<(NodeId, f64)> = order.iter().map(|id| (*id, scores[id])).collect(); + finalists.sort_by(|a, b| b.1.total_cmp(&a.1)); + finalists.truncate(MAX_FINALISTS); + let (top_id, top_score) = finalists + .into_iter() + .filter_map(|(id, s)| element(id).map(|el| (id, s * (1.0 - link_density(el))))) + .max_by(|a, b| a.1.total_cmp(&b.1))?; + let top = element(top_id)?; + if top.id() == body.id() { + return None; + } + + let mut roots = Vec::new(); + let threshold = (top_score * 0.2).max(10.0); + let parent = top.parent().and_then(ElementRef::wrap); + let siblings: Vec> = match parent { + Some(p) => p.children().filter_map(ElementRef::wrap).collect(), + None => vec![top], + }; + for sib in siblings { + if sib.id() == top.id() { + roots.push(ContentRoot { + el: sib, + whole: false, + }); + continue; + } + if is_unlikely(sib) || exclude.iter().any(|s| s.matches(&sib)) { + continue; + } + let keep = scores + .get(&sib.id()) + .is_some_and(|s| s * (1.0 - link_density(sib)) >= threshold) + || (sib.value().name() == "p" && is_prose(sib)); + if keep { + roots.push(ContentRoot { + el: sib, + whole: true, + }); + } + } + + let total: usize = roots.iter().map(|r| text_len(r.el)).sum(); + (total >= MIN_CONTENT_CHARS).then_some(roots) +} + +/// A paragraph that reads like prose: long enough, few links, or a short sentence. +fn is_prose(p: ElementRef<'_>) -> bool { + let len = text_len(p); + let density = link_density(p); + (len > 80 && density < 0.25) + || (len > 0 && density == 0.0 && p.text().any(|t| t.contains(". ") || t.ends_with('.'))) +} + +fn is_paragraph_like(el: ElementRef<'_>) -> bool { + match el.value().name() { + "p" | "pre" | "blockquote" => true, + "td" | "div" | "section" => !el.children().filter_map(ElementRef::wrap).any(is_block), + _ => false, + } +} + +fn is_block(el: ElementRef<'_>) -> bool { + matches!( + el.value().name(), + "address" + | "article" + | "aside" + | "blockquote" + | "dd" + | "div" + | "dl" + | "dt" + | "figure" + | "footer" + | "form" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + | "header" + | "hr" + | "li" + | "main" + | "nav" + | "ol" + | "p" + | "pre" + | "section" + | "table" + | "ul" + ) +} + +fn initial_score(el: ElementRef<'_>) -> f64 { + let tag = match el.value().name() { + "article" | "main" => 10.0, + "div" | "section" => 5.0, + "pre" | "td" | "blockquote" => 3.0, + "address" | "ol" | "ul" | "dl" | "dd" | "dt" | "li" | "form" | "aside" => -3.0, + "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "th" => -5.0, + _ => 0.0, + }; + tag + class_weight(el) +} + +const POSITIVE: &[&str] = &[ + "article", "body", "content", "entry", "hentry", "h-entry", "main", "page", "post", "text", + "blog", "story", +]; +const NEGATIVE: &[&str] = &[ + "-ad-", + "hidden", + "banner", + "combx", + "comment", + "com-", + "contact", + "footer", + "gdpr", + "masthead", + "media", + "meta", + "outbrain", + "promo", + "related", + "scroll", + "share", + "shoutbox", + "sidebar", + "skyscraper", + "sponsor", + "shopping", + "tags", + "widget", +]; +const UNLIKELY: &[&str] = &[ + "-ad-", + "ad-break", + "agegate", + "banner", + "breadcrumb", + "combx", + "comment", + "community", + "cookie", + "cover-wrap", + "disqus", + "extra", + "footer", + "gdpr", + "header", + "legends", + "menu", + "newsletter", + "pager", + "pagination", + "popup", + "promo", + "related", + "remark", + "replies", + "rss", + "share", + "shoutbox", + "sidebar", + "skyscraper", + "social", + "sponsor", + "subscribe", + "supplemental", + "yom-remote", +]; +/// Overrides [`UNLIKELY`] when present (e.g. `main-content-header` is kept). +const MAYBE: &[&str] = &[ + "article", "body", "column", "content", "main", "shadow", "post", "entry", "story", +]; +const UNLIKELY_TOKENS: &[&str] = &["ad", "ads", "nav", "navbar", "toolbar"]; +const UNLIKELY_ROLES: &[&str] = &[ + "menu", + "menubar", + "complementary", + "navigation", + "alert", + "alertdialog", + "dialog", +]; + +/// Lowercased `class` + `id`. +fn class_and_id(el: ElementRef<'_>) -> String { + let v = el.value(); + let mut s = String::new(); + for part in [v.attr("class"), v.attr("id")].into_iter().flatten() { + s.push(' '); + s.push_str(&part.to_ascii_lowercase()); + } + s +} + +fn class_weight(el: ElementRef<'_>) -> f64 { + let v = el.value(); + let mut w = 0.0; + for part in [v.attr("class"), v.attr("id")].into_iter().flatten() { + let part = part.to_ascii_lowercase(); + if NEGATIVE.iter().any(|n| part.contains(n)) { + w -= 25.0; + } + if POSITIVE.iter().any(|p| part.contains(p)) { + w += 25.0; + } + } + w +} + +pub(crate) fn is_unlikely(el: ElementRef<'_>) -> bool { + let v = el.value(); + if matches!(v.name(), "body" | "html" | "article" | "main" | "a") { + return false; + } + if v.attr("role") + .is_some_and(|r| UNLIKELY_ROLES.contains(&r.trim().to_ascii_lowercase().as_str())) + { + return true; + } + let s = class_and_id(el); + if s.is_empty() { + return false; + } + let tokens = || { + s.split(|c: char| !c.is_ascii_alphanumeric()) + .filter(|t| !t.is_empty()) + }; + let unlikely = + UNLIKELY.iter().any(|u| s.contains(u)) || tokens().any(|t| UNLIKELY_TOKENS.contains(&t)); + unlikely && !MAYBE.iter().any(|m| s.contains(m)) +} + +/// Visible text length in chars, whitespace runs counted once. +fn text_len(el: ElementRef<'_>) -> usize { + let mut n = 0; + let mut space = true; + for t in el.text() { + for c in t.chars() { + if c.is_whitespace() { + if !space { + n += 1; + } + space = true; + } else { + n += 1; + space = false; + } + } + } + n +} + +/// Share of `el`'s text inside links (`#` fragment links count 30%, as in Readability). +fn link_density(el: ElementRef<'_>) -> f64 { + let total = text_len(el); + if total == 0 { + return 0.0; + } + let mut linked = 0.0; + for node in el.descendants() { + if let Node::Element(e) = node.value() + && e.name() == "a" + && let Some(a) = ElementRef::wrap(node) + { + let coeff = if e + .attr("href") + .is_some_and(|h| h.trim_start().starts_with('#')) + { + 0.3 + } else { + 1.0 + }; + linked += text_len(a) as f64 * coeff; + } + } + (linked / total as f64).min(1.0) +} + +#[cfg(test)] +mod tests { + use super::*; + use scraper::Html; + + fn el_with(attrs: &str) -> bool { + let dom = Html::parse_fragment(&format!("
x
")); + let sel = Selector::parse("div").unwrap(); + is_unlikely(dom.select(&sel).next().unwrap()) + } + + #[test] + fn unlikely_classification() { + assert!(el_with(r#"class="sidebar""#)); + assert!(el_with(r#"id="comments""#)); + assert!(el_with(r#"class="top ads""#)); + assert!(el_with(r#"role="Navigation""#)); + assert!( + !el_with(r#"class="main-sidebar-content""#), + "MAYBE overrides" + ); + assert!(!el_with(r#"class="headline""#)); + assert!(!el_with(r#"class="loading""#), "'ad' only as a whole token"); + assert!(!el_with("")); + } + + #[test] + fn text_len_collapses_whitespace() { + let dom = Html::parse_fragment("

a \n\n b

"); + let p = dom.select(&Selector::parse("p").unwrap()).next().unwrap(); + assert_eq!(text_len(p), 4); // leading space suppressed: "a b " + } + + #[test] + fn link_density_counts_fragment_links_less() { + let dom = Html::parse_fragment( + r##""##, + ); + let d = dom.select(&Selector::parse("div").unwrap()).next().unwrap(); + assert!((link_density(d) - (5.0 + 3.0) / 20.0).abs() < 1e-9); + } +} diff --git a/crates/markkit/tests/fixtures/html/news_page.html b/crates/markkit/tests/fixtures/html/news_page.html new file mode 100644 index 0000000..498d38b --- /dev/null +++ b/crates/markkit/tests/fixtures/html/news_page.html @@ -0,0 +1,46 @@ + + + + Rust 2026 edition lands | Example News + + + +
+ Home Tech Science Log in +
+ +
+
+
+

What changed

+

The Rust project shipped its 2026 edition on Thursday, bringing the largest set of + language changes since 2021, according to the release team, which spent two years on it.

+

Async closures, generators, and a reworked borrow checker are the headline features, + and the team says migration is automatic for most crates, with a few manual steps left.

+ +

Library authors can opt in per crate, so mixed-edition dependency graphs keep working, + a guarantee the project has kept since the first edition shipped in 2018.

+ 
edition = "2026"
+

The full release notes, including a migration guide, are available on the project blog.

+
+
+
+

Most read

+ +
+
+
+

Comments

+

First comment, this is great news, finally, I have waited for generators for years.

+
+
+

Related: the 2024 edition retrospective, one year later

+
+ + diff --git a/crates/markkit/tests/fixtures/html/news_page.main.md b/crates/markkit/tests/fixtures/html/news_page.main.md new file mode 100644 index 0000000..c9bd59c --- /dev/null +++ b/crates/markkit/tests/fixtures/html/news_page.main.md @@ -0,0 +1,13 @@ +## What changed + +The Rust project shipped its 2026 edition on Thursday, bringing the largest set of language changes since 2021, according to the release team, which spent two years on it. + +Async closures, generators, and a reworked borrow checker are the headline features, and the team says migration is automatic for most crates, with a few manual steps left. + +Library authors can opt in per crate, so mixed-edition dependency graphs keep working, a guarantee the project has kept since the first edition shipped in 2018. + +```toml +edition = "2026" +``` + +The full release notes, including a migration guide, are available on the project blog. diff --git a/crates/markkit/tests/html_fixtures.rs b/crates/markkit/tests/html_fixtures.rs index ea69005..d221276 100644 --- a/crates/markkit/tests/html_fixtures.rs +++ b/crates/markkit/tests/html_fixtures.rs @@ -64,3 +64,12 @@ fn fixture_full_article() { fn fixture_malformed() { fixture_test("malformed"); } + +#[test] +fn fixture_news_page_main_content() { + let html = fs::read_to_string("tests/fixtures/html/news_page.html").unwrap(); + let expected = fs::read_to_string("tests/fixtures/html/news_page.main.md").unwrap(); + let m = Markkit::builder().main_content(true).build(); + let actual = m.extract_html_sync(&html).unwrap(); + assert_eq!(actual, expected, "\n--- Actual ---\n{actual}"); +} diff --git a/crates/markkit/tests/html_main_content.rs b/crates/markkit/tests/html_main_content.rs new file mode 100644 index 0000000..fe80b41 --- /dev/null +++ b/crates/markkit/tests/html_main_content.rs @@ -0,0 +1,167 @@ +//! Main-content detection (`MarkkitBuilder::main_content`). + +#![cfg(feature = "html")] + +use std::time::{Duration, Instant}; + +use markkit::Markkit; + +fn main(html: &str) -> String { + Markkit::builder() + .main_content(true) + .build() + .extract_html_sync(html) + .unwrap() +} + +const PROSE: &str = "This paragraph is ordinary article prose, long enough to score, with commas, \ + clauses, and a full stop at the end of it."; + +fn news_page() -> String { + std::fs::read_to_string("tests/fixtures/html/news_page.html").unwrap() +} + +#[test] +fn off_by_default_keeps_boilerplate() { + let md = Markkit::builder() + .build() + .extract_html_sync(&news_page()) + .unwrap(); + assert!(md.contains("Most read")); + assert!(md.contains("First comment")); +} + +#[test] +fn drops_sidebar_comments_share_and_banner() { + let md = main(&news_page()); + for noise in [ + "Most read", + "First comment", + "Share on X", + "Subscribe today", + "Log in", + "Related:", + ] { + assert!(!md.contains(noise), "{noise} leaked:\n{md}"); + } + assert!(md.contains("edition = \"2026\"")); +} + +#[test] +fn classless_layout_prefers_prose_over_link_lists() { + let links: String = (0..12) + .map(|i| { + format!("

A link teaser headline number {i}, read more

") + }) + .collect(); + let html = format!( + "
{links}

{PROSE}

{PROSE}

{PROSE}

" + ); + let md = main(&html); + assert!(md.contains("ordinary article prose")); + assert!(!md.contains("teaser"), "{md}"); +} + +#[test] +fn article_siblings_with_prose_are_kept() { + let html = format!( + "

{PROSE}

{PROSE}

\ +

A short closing sentence after the article.

\ +

Next story link

" + ); + let md = main(&html); + assert!(md.contains("closing sentence"), "{md}"); + assert!(!md.contains("Next story"), "{md}"); +} + +#[test] +fn short_page_falls_back_to_full_body() { + assert_eq!(main("

Hello

world
"), "Hello\n\nworld\n"); + assert_eq!(main(""), ""); +} + +#[test] +fn include_selectors_win() { + let md = Markkit::builder() + .main_content(true) + .include_selector(".sidebar") + .build() + .extract_html_sync(&news_page()) + .unwrap(); + assert!(md.contains("Most read")); + assert!(!md.contains("What changed")); +} + +// ─── Untrusted input ──────────────────────────────────────────────── + +#[test] +fn deep_nesting_does_not_overflow() { + let html = format!( + "{}

{PROSE}

{}", + "
".repeat(5_000), + "
".repeat(5_000) + ); + let md = main(&html); + assert!(md.contains("ordinary article prose")); +} + +#[test] +fn many_paragraphs_stay_linear() { + let html = format!( + "
{}
", + format!("

{PROSE}

").repeat(20_000) + ); + let start = Instant::now(); + let md = main(&html); + assert!(md.len() > 1_000_000); + assert!( + start.elapsed() < Duration::from_secs(20), + "{:?}", + start.elapsed() + ); +} + +#[test] +fn many_sibling_containers_stay_bounded() { + let block = format!("

{PROSE}

"); + let html = format!("{}", block.repeat(5_000)); + let start = Instant::now(); + let md = main(&html); + assert!(md.contains("ordinary article prose")); + assert!( + start.elapsed() < Duration::from_secs(20), + "{:?}", + start.elapsed() + ); +} + +#[test] +fn non_content_elements_still_stripped() { + let html = format!( + "

{PROSE}

{PROSE}

\ +
" + ); + let md = main(&html); + assert!(!md.contains("alert") && !md.contains("`; subtrees matching exclude selectors or "unlikely" + class/id/role (`sidebar`, `comment`, `share`, `related`, `banner`, `cookie`, tokens `ad`/`nav`, + roles `navigation`/`complementary`/`dialog`...) are skipped unless the class also says + `content`/`article`/`main`/`post`/... + 2. Paragraph-like blocks (`p`, `pre`, `blockquote`, leaf `div`/`section`/`td`) with >= 25 chars + score `1 + commas + min(floor(chars/100), 3)`, credited to parent (1), grandparent (1/2), + great-grandparent (1/3). Ancestors start from a tag score (`article`/`main` +10, `div` +5, + lists -3, headings -5) plus +-25 for positive/negative class/id words. + 3. Top 10 by raw score are scaled by `1 - link density` (fragment links count 30%); best wins. + 4. Winner's siblings are kept when their scaled score >= max(10, 0.2 x winner), or when a + `

` reads like prose (> 80 chars and link density < 0.25, or a link-free sentence). + 5. Under 140 chars of chosen text, no candidate, or `` itself winning: fall back to the + full body. Inside the chosen content, unlikely subtrees are dropped as well. + Headings above the article container (page `

`) are usually outside it; the title is in + `Extraction::title`. Fixture: `news_page.html` -> `news_page.main.md`. ## Page Metadata diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index ca43cfa..d81cd77 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -69,6 +69,7 @@ Via `MarkkitBuilder`: - URL fetch policy (`allow_private_networks`, `allow_host`, `max_redirects`) - Base URL for relative links, images and metadata URLs (`base_url`) - Provenance (`track_sources`): per-block `Source` in the IR (off by default) +- Main-content detection for HTML (`main_content`) ## Public entry points diff --git a/knowledge/log.md b/knowledge/log.md index 63963b1..ff3da39 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-26 +* **Main content**: opt-in Readability-style detection (`main_content`, CLI `--main-content`) keeps the article body and drops sidebars, comments, share bars; falls back to the full body when nothing scores. Algorithm in [HTML Extraction](formats/html-extraction.md); bounds in [Threat Model](security/threat-model.md) T2g. + * **Relative URLs**: links, images and metadata URLs resolve against `base_url` (or the final `extract_url` response URL) plus ``. Only scheme-less references change; page bases are limited to `http(s)`. New `extract_url_document`. See [HTML Extraction](formats/html-extraction.md) and [Threat Model](security/threat-model.md) T1. * **Transforms**: `transform::Transform` passes between parse and render (`MarkkitBuilder::transform`), built-ins `NormalizeHeadings` / `DropImages`, `from_fn`, IR walkers. `Error` is now `#[non_exhaustive]` with `Error::Transform`. See [Markkit Architecture](foundations/architecture.md), [Examples](operations/examples.md). diff --git a/knowledge/security/security-model.md b/knowledge/security/security-model.md index 7268562..47789df 100644 --- a/knowledge/security/security-model.md +++ b/knowledge/security/security-model.md @@ -16,6 +16,7 @@ Markkit processes untrusted input from the internet. Key threats: ### TM-INJ: Injection via extracted content - **Mitigation**: All output is markdown text; no code execution - **Mitigation**: Strip script/style/event handler content +- **Mitigation**: Metadata URL fields drop non-http(s) schemes; base resolution never rewrites absolute URLs and a page `` must be http(s); see [Threat Model](threat-model.md) T1 ### TM-DOS: Denial of service - **Mitigation**: Input size limits @@ -23,6 +24,7 @@ Markkit processes untrusted input from the internet. Key threats: - **Mitigation**: Output size limits - **Mitigation**: Nesting depth limits - **Mitigation**: Table span layout bounded by a grid budget (spans dropped past it); see [Threat Model](threat-model.md) T2e +- **Mitigation**: JSON-LD metadata bounded (script count/size, node count, nesting) and main-content scoring linear and non-recursive; see [Threat Model](threat-model.md) T2f, T2g ### TM-SSRF: Server-side request forgery (URL fetching) - **Mitigation**: Non-public addresses refused by default, after DNS resolution and on every redirect (opt in with `allow_private_networks`) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 6e322d2..5fb087e 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -97,6 +97,12 @@ tags: - Tests: `html_metadata::deeply_nested_json_ld_does_not_overflow`, `oversized_json_ld_skipped`, `many_json_ld_scripts_bounded`, `malformed_json_ld_ignored` +#### T2g: Main-Content Scoring +- One iterative DOM traversal (no recursion); scored paragraphs are leaf blocks, so text + measurement is linear; link density only for the top 10 candidates and the winner's siblings +- Tests: `html_main_content::deep_nesting_does_not_overflow`, `many_paragraphs_stay_linear`, + `many_sibling_containers_stay_bounded`, `non_content_elements_still_stripped` + **Test Coverage**: - `html_security::deeply_nested_elements` - `html_security::very_long_text_node`