diff --git a/.gitignore b/.gitignore index 211833a..73f8f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ crates/markkit-js/index.js crates/markkit-js/index.d.ts crates/markkit-js/package-lock.json crates/markkit-js/*.tgz + +# mira saved eval runs (see mira.toml) +crates/markkit-eval/results/*/ diff --git a/AGENTS.md b/AGENTS.md index 328972f..713eefe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | operations/maintenance | Maintenance invocation contract + checklist + workflow credential audit | | operations/release-process | Agent-driven release, crates.io + PyPI + npm | | operations/performance-results | Benchmark harness, metrics, committed results | -| operations/eval | LLM-as-judge eval design (harness is a stub) | +| operations/eval | mira eval study: dataset, checks, metrics, baseline | ### Documentation @@ -86,7 +86,7 @@ just pre-pr # Pre-PR checks | `markkit` | Core extraction library (HTML, DOCX, spreadsheets) | | `markkit-cli` | `markkit` CLI | | `markkit-bench` | Benchmark harness: ops/s, throughput, allocations; results committed | -| `markkit-eval` | LLM evaluation harness (stub) | +| `markkit-eval` | Extraction-quality eval, mira study (`just eval`) | | `markkit-python` | Python bindings (PyO3), module `markkit` | | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` | | `markkit-wasm` | Browser/edge WebAssembly (wasm-bindgen), package `@everruns/markkit-wasm` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 81fa83e..9fabe68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. Block variants gain a `source` field (use `..` in patterns, `Node::paragraph`/`Node::heading` to construct). DOCX `gridSpan` is carried as `colspan`. +- `markkit-eval` is now a [mira](https://github.com/everruns/mira) eval study. `just eval` runs + `markkit_extract`, an offline extraction-quality eval over ten realistic HTML pages scored for + completeness, accuracy, formatting and noise (needs `cargo install mira-cli`). A `mode` axis + compares default extraction with `main_content`; dated results are committed in + `crates/markkit-eval/results/`. - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. - CI runs every example; criterion benchmarks run with `just bench-criterion`. diff --git a/Cargo.lock b/Cargo.lock index ed873b7..b72c1e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,17 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "atoi_simd" version = "0.18.1" @@ -1078,6 +1089,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -1284,8 +1304,10 @@ dependencies = [ name = "markkit-eval" version = "0.1.1" dependencies = [ - "anyhow", - "clap", + "markkit", + "mira-eval", + "serde", + "serde_json", "tokio", ] @@ -1358,6 +1380,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mira-eval" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75db8c713ee4b8ed2fd8eb1797dd3129b0e545a039e9cef45ac19458447cc16c" +dependencies = [ + "async-trait", + "inventory", + "mira-macros", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "mira-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfd4d98e649e99b006acace5360e65ca6fd0bc6593e6502514592316ab38530" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -2317,6 +2366,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -2535,6 +2594,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/crates/markkit-eval/Cargo.toml b/crates/markkit-eval/Cargo.toml index c8d5062..657ab01 100644 --- a/crates/markkit-eval/Cargo.toml +++ b/crates/markkit-eval/Cargo.toml @@ -1,5 +1,10 @@ -# LLM evaluation harness for markkit extraction quality -# See knowledge/operations/eval.md for design +# markkit-eval: mira eval study for markkit extraction quality. +# Reimplemented on the mira eval framework (github.com/everruns/mira): +# markkit extraction runs as a mira `Subject`; dataset pages are mira +# `Sample`s; deterministic expectation checks run as mira `Scorer`s, one per +# quality metric. The `mira` CLI hosts this binary over stdio and owns +# selection, scheduling, saved runs, and reporting. +# See knowledge/operations/eval.md for design decisions. [package] name = "markkit-eval" @@ -7,13 +12,21 @@ version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "LLM evaluation harness for markkit extraction quality" +description = "Mira eval study for markkit extraction quality" +publish = false [[bin]] name = "markkit-eval" path = "src/main.rs" [dependencies] +# Mira eval framework (crates.io), imported as `mira`. `macros` provides the +# #[eval] discovery attribute. In-process subject only: no mira-everruns. +mira-eval = { version = "0.5", features = ["macros"] } +# HTML only, no HTTP client: the eval never fetches (pages are embedded). +markkit = { path = "../markkit", default-features = false, features = ["html"] } +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -clap.workspace = true -anyhow.workspace = true diff --git a/crates/markkit-eval/data/extract-tasks.jsonl b/crates/markkit-eval/data/extract-tasks.jsonl new file mode 100644 index 0000000..69f3c52 --- /dev/null +++ b/crates/markkit-eval/data/extract-tasks.jsonl @@ -0,0 +1,11 @@ +# markkit_extract dataset: one task per line. Schema and check types: knowledge/operations/eval.md +{"id": "news_article", "category": "news", "description": "News article with cookie banner, site nav, inline ad, newsletter form, related-stories aside and footer", "page": "news_article.html", "expectations": [{"check": "heading", "level": 1, "text": "City Council Approves Riverside Park Expansion", "weight": 2}, {"check": "contains", "text": "voted 7–2 on Tuesday to expand Riverside Park"}, {"check": "contains", "text": "**18 acres**"}, {"check": "contains", "text": "$14.2 million"}, {"check": "line", "text": "> This is the largest addition to our public green space in fifty years."}, {"check": "contains", "text": "*spring 2028*"}, {"check": "link", "text": "city parks website", "url": "https://harbor.test/parks/riverside"}, {"check": "heading", "level": 2, "text": "What happens to the rail yard"}, {"check": "contains", "text": "two historic signal towers"}, {"check": "contains", "text": "![Aerial view of Riverside Park](https://cdn.harborgazette.test/img/riverside-park.jpg)"}, {"check": "order", "first": "$14.2 million", "then": "signal towers"}, {"check": "not_contains", "text": "We use cookies"}, {"check": "not_contains", "text": "Accept all cookies"}, {"check": "not_contains", "text": "Subscribe"}, {"check": "not_contains", "text": "Refinance today"}, {"check": "not_contains", "text": "Advertisement"}, {"check": "not_contains", "text": "Related stories"}, {"check": "not_contains", "text": "New bike lanes"}, {"check": "not_contains", "text": "All rights reserved"}, {"check": "not_contains", "text": "dataLayer"}, {"check": "not_contains", "text": "Get the morning briefing"}]} +{"id": "docs_page", "category": "docs", "description": "Docs page with sidebar nav, option table, TOML and Rust code blocks, callout, edit link and pager", "page": "docs_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Configuration", "weight": 2}, {"check": "contains", "text": "`tideline.toml`"}, {"check": "line", "text": "| Key | Type | Default | Description |"}, {"check": "line", "text": "| `workers` | integer | 4 | Number of worker threads |"}, {"check": "line", "text": "| `retry` | boolean | true | Retry failed requests once |"}, {"check": "line", "text": "```toml"}, {"check": "line", "text": "```rust"}, {"check": "contains", "text": "let cfg = Config::load(\"tideline.toml\")?;"}, {"check": "contains", "text": "fn main() -> anyhow::Result<()> {"}, {"check": "contains", "text": "unknown keys are rejected at startup"}, {"check": "link", "text": "environment variables", "url": "/docs/env"}, {"check": "order", "first": "## Options", "then": "## Example"}, {"check": "order", "first": "## Example", "then": "## Loading from code"}, {"check": "not_contains", "text": "Getting started"}, {"check": "not_contains", "text": "Quickstart"}, {"check": "not_contains", "text": "Search docs"}, {"check": "not_contains", "text": "Edit this page"}, {"check": "not_contains", "text": "Last updated"}, {"check": "not_contains", "text": "Previous:"}, {"check": "not_contains", "text": "Built with Tideline Docs"}]} +{"id": "blog_post", "category": "blog", "description": "Engineering blog post with share buttons, author bio, comment section and newsletter popup", "page": "blog_post.html", "expectations": [{"check": "heading", "level": 1, "text": "Why we moved our build to Bazel", "weight": 2}, {"check": "contains", "text": "1.4 million lines of code"}, {"check": "contains", "text": "**47 minutes**"}, {"check": "heading", "level": 2, "text": "What we tried first"}, {"check": "line", "text": "1. Caching `node_modules` between CI runs"}, {"check": "line", "text": "3. Upgrading CI runners to 32 cores"}, {"check": "heading", "level": 2, "text": "Results"}, {"check": "contains", "text": "92% of builds hit the remote cache"}, {"check": "order", "first": "What we tried first", "then": "Results"}, {"check": "not_contains", "text": "We're hiring"}, {"check": "not_contains", "text": "Share on X"}, {"check": "not_contains", "text": "Share on LinkedIn"}, {"check": "not_contains", "text": "14 Comments"}, {"check": "not_contains", "text": "Did you consider Buck2"}, {"check": "not_contains", "text": "Log in to comment"}, {"check": "not_contains", "text": "Subscribe to our newsletter"}]} +{"id": "product_page", "category": "commerce", "description": "Product page with nav, breadcrumbs, buy box form, spec table, reviews, recommendations and promo banner", "page": "product_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Northwind Trail 40L Backpack", "weight": 2}, {"check": "contains", "text": "$189.00"}, {"check": "contains", "text": "ventilated back panel and a removable rain cover"}, {"check": "line", "text": "- Recycled 210D ripstop nylon"}, {"check": "line", "text": "- Hydration sleeve fits a 3 L reservoir"}, {"check": "contains", "text": "Adjustable torso length (15–21 in)"}, {"check": "contains", "text": "1.2 kg"}, {"check": "contains", "text": "64 × 32 × 24 cm"}, {"check": "line", "text": "| Weight | 1.2 kg |"}, {"check": "contains", "text": "4.6 out of 5 stars"}, {"check": "order", "first": "## Features", "then": "## Specifications"}, {"check": "not_contains", "text": "Cart (0)"}, {"check": "not_contains", "text": "Add to cart"}, {"check": "not_contains", "text": "Home / "}, {"check": "not_contains", "text": "You may also like"}, {"check": "not_contains", "text": "Ridgeline 30L"}, {"check": "not_contains", "text": "TRAIL10"}, {"check": "not_contains", "text": "1-800-555-0199"}]} +{"id": "recipe", "category": "recipe", "description": "Recipe with sponsored ad slots, ingredient list, numbered steps and a rating widget", "page": "recipe.html", "expectations": [{"check": "heading", "level": 1, "text": "Weeknight Lemon Garlic Pasta", "weight": 2}, {"check": "heading", "level": 2, "text": "Ingredients"}, {"check": "heading", "level": 2, "text": "Instructions"}, {"check": "line", "text": "- 350 g spaghetti"}, {"check": "line", "text": "- 40 g grated parmesan"}, {"check": "line", "text": "1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water."}, {"check": "line", "text": "4. Loosen with pasta water as needed and serve immediately."}, {"check": "contains", "text": "Serves: 4"}, {"check": "contains", "text": "replace the parmesan with toasted breadcrumbs"}, {"check": "order", "first": "Warm the olive oil", "then": "Toss the pasta"}, {"check": "not_contains", "text": "Sponsored"}, {"check": "not_contains", "text": "MealBox"}, {"check": "not_contains", "text": "Chef's knife"}, {"check": "not_contains", "text": "Rate this recipe"}, {"check": "not_contains", "text": "Follow us on"}]} +{"id": "wiki_article", "category": "reference", "description": "Encyclopedia article with infobox, inline citations, table of contents, edit links, categories and footer", "page": "wiki_article.html", "expectations": [{"check": "heading", "level": 1, "text": "Lighthouse of Alexandria", "weight": 2}, {"check": "contains", "text": "built by the Ptolemaic Kingdom on the island of Pharos"}, {"check": "contains", "text": "Seven Wonders of the Ancient World"}, {"check": "heading", "level": 2, "text": "Construction"}, {"check": "heading", "level": 3, "text": "Design"}, {"check": "heading", "level": 2, "text": "Destruction"}, {"check": "contains", "text": "Sostratus of Cnidus"}, {"check": "contains", "text": "between 956 and 1323 AD"}, {"check": "contains", "text": "c. 280 BC"}, {"check": "order", "first": "Construction was completed", "then": "badly damaged by earthquakes"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Random article"}, {"check": "not_contains", "text": "[edit]"}, {"check": "not_contains", "text": "From OpenWiki, the free encyclopedia"}, {"check": "not_contains", "text": "Contents"}, {"check": "not_contains", "text": "Categories:"}, {"check": "not_contains", "text": "last edited on"}]} +{"id": "forum_thread", "category": "forum", "description": "Q&A thread with vote widgets, tags, accepted answer with code, user cards and sidebar", "page": "forum_thread.html", "expectations": [{"check": "heading", "level": 1, "text": "How do I reverse a linked list in place?", "weight": 2}, {"check": "contains", "text": "reverse it without allocating a new list"}, {"check": "line", "text": "```python"}, {"check": "contains", "text": " nxt = node.next"}, {"check": "contains", "text": "def reverse(head):"}, {"check": "contains", "text": "O(n) time and O(1) extra space"}, {"check": "contains", "text": "recursive version also works"}, {"check": "order", "first": "without allocating a new list", "then": "You overwrite"}, {"check": "order", "first": "O(1) extra space", "then": "recursive version"}, {"check": "not_contains", "text": "Viewed 48k times"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Hot Network Questions"}, {"check": "not_contains", "text": "CSS grid"}, {"check": "not_contains", "text": "user contributions licensed"}, {"check": "not_contains", "text": "▲"}]} +{"id": "changelog", "category": "changelog", "description": "Release notes with nested lists, strikethrough, issue links and call-to-action box", "page": "changelog.html", "expectations": [{"check": "heading", "level": 1, "text": "Changelog", "weight": 2}, {"check": "heading", "level": 2, "text": "v2.4.0 — 2026-04-02"}, {"check": "heading", "level": 3, "text": "Added"}, {"check": "heading", "level": 3, "text": "Fixed"}, {"check": "heading", "level": 2, "text": "v2.3.1 — 2026-03-10"}, {"check": "line", "text": "- Offline mode for the desktop editor"}, {"check": "contains", "text": " - Custom cover images"}, {"check": "link", "text": "#1182", "url": "https://git.test/quillstack/issues/1182"}, {"check": "contains", "text": "~~Spellcheck ignored code blocks~~"}, {"check": "contains", "text": "Sanitize SVG uploads to prevent stored XSS"}, {"check": "order", "first": "v2.4.0", "then": "v2.3.1"}, {"check": "not_contains", "text": "Pricing"}, {"check": "not_contains", "text": "Start your free trial"}, {"check": "not_contains", "text": "Status"}]} +{"id": "data_table", "category": "data", "description": "Data report with a captioned table including thead, tbody and tfoot", "page": "data_table.html", "expectations": [{"check": "heading", "level": 1, "text": "Electricity Generation by Source, 2025", "weight": 2}, {"check": "contains", "text": "terawatt-hours (TWh)"}, {"check": "line", "text": "| Source | Q1 | Q2 | Q3 | Q4 |", "weight": 2}, {"check": "line", "text": "| Wind | 21.4 | 15.2 | 12.8 | 24.9 |"}, {"check": "line", "text": "| Nuclear | 12.0 | 11.6 | 11.9 | 12.1 |"}, {"check": "contains", "text": "Natural gas", "metric": "completeness"}, {"check": "contains", "text": "Total"}, {"check": "contains", "text": "72.4"}, {"check": "contains", "text": "Quarterly generation"}, {"check": "order", "first": "| Wind |", "then": "| Solar |"}, {"check": "link", "text": "Download the data (CSV)", "url": "/reports/2025-energy.csv"}, {"check": "not_contains", "text": "Reports"}, {"check": "not_contains", "text": "Share:"}, {"check": "not_contains", "text": "independent project"}]} +{"id": "legacy_layout", "category": "legacy", "description": "1990s table-layout page with font tags, unclosed paragraphs, line breaks and a hit counter", "page": "legacy_layout.html", "expectations": [{"check": "contains", "text": "Pine Hollow Model Railroad Club", "weight": 2}, {"check": "contains", "text": "**Tuesday at 7 PM**"}, {"check": "contains", "text": "Grange Hall, 18 Mill Road"}, {"check": "contains", "text": "600 feet of track"}, {"check": "contains", "text": "Boston & Maine railroad in 1952"}, {"check": "contains", "text": "Annual dues are $40."}, {"check": "link", "text": "walt@pinehollow.test", "url": "mailto:walt@pinehollow.test"}, {"check": "order", "first": "Welcome to the home", "then": "New members are always welcome"}, {"check": "not_contains", "text": "MENU", "weight": 2}, {"check": "not_contains", "text": "Guestbook"}, {"check": "not_contains", "text": "visitor number"}, {"check": "not_contains", "text": "Netscape Navigator"}]} diff --git a/crates/markkit-eval/data/pages/blog_post.html b/crates/markkit-eval/data/pages/blog_post.html new file mode 100644 index 0000000..06e6115 --- /dev/null +++ b/crates/markkit-eval/data/pages/blog_post.html @@ -0,0 +1,38 @@ + + +Why we moved our build to Bazel - Lattice Engineering + +
Lattice Engineering Blog | We're hiring!
+
+

Why we moved our build to Bazel

+
Posted by Jordan Alvarez on 9 January 2026 · 6 min read
+
+ Share on X + Share on LinkedIn +
+

Our monorepo had grown to 1.4 million lines of code, and a clean CI build took 47 minutes. Incremental builds were unreliable, so engineers ran clean builds out of habit.

+

What we tried first

+
    +
  1. Caching node_modules between CI runs
  2. +
  3. Splitting the test suite across 12 shards
  4. +
  5. Upgrading CI runners to 32 cores
  6. +
+

Each change helped a little, but none fixed the root problem: the build had no accurate dependency graph.

+

Results

+

After the migration, the median CI build takes 6 minutes, and 92% of builds hit the remote cache.

+

Thanks to the developer productivity team for leading the migration.

+
+ Jordan Alvarez +

Jordan Alvarez is a staff engineer on the platform team.

+
+
+
+

14 Comments

+
devops_dan said: Great write-up! Did you consider Buck2?
+
sarahk said: How long did the migration take?
+ Log in to comment +
+ + + + diff --git a/crates/markkit-eval/data/pages/changelog.html b/crates/markkit-eval/data/pages/changelog.html new file mode 100644 index 0000000..21b7088 --- /dev/null +++ b/crates/markkit-eval/data/pages/changelog.html @@ -0,0 +1,38 @@ + + +Changelog - Quillstack + +
+
+

Changelog

+

All notable changes to Quillstack are documented here. Subscribe via RSS.

+
+

v2.4.0 — 2026-04-02

+

Added

+ +

Fixed

+ +
+
+

v2.3.1 — 2026-03-10

+

Security

+ +
+

Ready to write? Start your free trial

+
+ + + diff --git a/crates/markkit-eval/data/pages/data_table.html b/crates/markkit-eval/data/pages/data_table.html new file mode 100644 index 0000000..357081b --- /dev/null +++ b/crates/markkit-eval/data/pages/data_table.html @@ -0,0 +1,30 @@ + + +Quarterly Energy Report - Grid Watch + +
+
+

Electricity Generation by Source, 2025

+

Figures are in terawatt-hours (TWh). Source: national grid operator, published February 2026.

+ + + + + + + + + + + + + + +
Quarterly generation
SourceQ1Q2Q3Q4
Wind21.415.212.824.9
Solar4.111.713.33.6
Natural gas30.222.526.031.8
Nuclear12.011.611.912.1
Total67.761.064.072.4
+

Wind overtook natural gas in no quarter, but the gap narrowed to 6.9 TWh in Q4.

+

Download the data (CSV)

+
+
Share: X Email
+ + + diff --git a/crates/markkit-eval/data/pages/docs_page.html b/crates/markkit-eval/data/pages/docs_page.html new file mode 100644 index 0000000..2a78ebd --- /dev/null +++ b/crates/markkit-eval/data/pages/docs_page.html @@ -0,0 +1,60 @@ + + +Configuration - Tideline Docs + +
Tideline
+
+ +
+

Configuration

+

Tideline reads its settings from tideline.toml in the project root. Every key is optional.

+

Options

+ + + + + + + + + +
KeyTypeDefaultDescription
workersinteger4Number of worker threads
timeoutstring30sRequest timeout
retrybooleantrueRetry failed requests once
+

Example

+
[server]
+workers = 8
+timeout = "45s"
+
+[cache]
+enabled = true
+
+

Loading from code

+

Use Config::load to read the file manually:

+
use tideline::Config;
+
+fn main() -> anyhow::Result<()> {
+    let cfg = Config::load("tideline.toml")?;
+    println!("{} workers", cfg.workers);
+    Ok(())
+}
+
+
+

Warning: unknown keys are rejected at startup.

+
+

See environment variables to override settings per deployment.

+
+ Edit this page on GitHub + Last updated on Feb 2, 2026 +
+ +
+
+ + + diff --git a/crates/markkit-eval/data/pages/forum_thread.html b/crates/markkit-eval/data/pages/forum_thread.html new file mode 100644 index 0000000..522a41d --- /dev/null +++ b/crates/markkit-eval/data/pages/forum_thread.html @@ -0,0 +1,46 @@ + + +How do I reverse a linked list in place? - DevAnswers + +
+
+

How do I reverse a linked list in place?

+
Asked 3 years ago · Viewed 48k times
+
312
+
+

I have a singly linked list and want to reverse it without allocating a new list. My attempt loses the rest of the list after the first node:

+
while node:
+    node.next = prev
+    prev = node
+    node = node.next
+
+
python linked-list
+
+
+

2 Answers

+
+
518 Accepted
+
+

You overwrite node.next before saving it. Keep a reference to the next node first:

+
def reverse(head):
+    prev = None
+    node = head
+    while node:
+        nxt = node.next
+        node.next = prev
+        prev = node
+        node = nxt
+    return prev
+

This runs in O(n) time and O(1) extra space.

+
+
answered by ada
+
+
+
41
+

A recursive version also works, but it uses O(n) stack space and can hit the recursion limit on long lists.

+
+
+ + + + diff --git a/crates/markkit-eval/data/pages/legacy_layout.html b/crates/markkit-eval/data/pages/legacy_layout.html new file mode 100644 index 0000000..ad6699f --- /dev/null +++ b/crates/markkit-eval/data/pages/legacy_layout.html @@ -0,0 +1,27 @@ + +Pine Hollow Model Railroad Club + + + + + + +
+MENU
+Home
+Our Layout
+Join
+Guestbook +
+
Pine Hollow Model Railroad Club
+

Welcome to the home of the Pine Hollow Model Railroad Club! We meet every Tuesday at 7 PM in the basement of the Grange Hall, 18 Mill Road. +

Our HO scale layout has over 600 feet of track and models the Boston & Maine railroad in 1952. +

New members are always welcome.
+Annual dues are $40.
+Contact our secretary, Walt Brennan, at walt@pinehollow.test. +


+

You are visitor number 004127 since 1998. +

Best viewed in Netscape Navigator 4.0 at 800x600. +

+ + diff --git a/crates/markkit-eval/data/pages/news_article.html b/crates/markkit-eval/data/pages/news_article.html new file mode 100644 index 0000000..27fe862 --- /dev/null +++ b/crates/markkit-eval/data/pages/news_article.html @@ -0,0 +1,61 @@ + + + + + City Council Approves Riverside Park Expansion | The Harbor Gazette + + + + + + +
+
+

City Council Approves Riverside Park Expansion

+ +
+ Aerial view of Riverside Park +
The current park covers 42 acres along the east bank.
+
+

The city council voted 7–2 on Tuesday to expand Riverside Park by 18 acres, converting a former rail yard into wetlands and walking trails.

+ +

The project is expected to cost $14.2 million, with most of the funding coming from a state conservation grant approved last year.

+
+

This is the largest addition to our public green space in fifty years.

+
+

Construction will begin in the fall and is scheduled to finish by spring 2028. Residents can review the plans at the city parks website.

+

What happens to the rail yard

+

Contaminated soil will be removed in the first phase. The two historic signal towers will be preserved as observation decks.

+ +
+ +
+ + + + diff --git a/crates/markkit-eval/data/pages/product_page.html b/crates/markkit-eval/data/pages/product_page.html new file mode 100644 index 0000000..721cb7e --- /dev/null +++ b/crates/markkit-eval/data/pages/product_page.html @@ -0,0 +1,40 @@ + + +Northwind Trail 40L Backpack | Summit Outfitters + +
+ +
+ +
+

Northwind Trail 40L Backpack

+

$189.00

+

A lightweight pack for multi-day hikes, with a ventilated back panel and a removable rain cover.

+
+ + +
+

Features

+ +

Specifications

+ + + + +
Weight1.2 kg
Capacity40 L
Dimensions64 × 32 × 24 cm
+

Customer reviews

+

4.6 out of 5 stars (212 reviews)

+

Comfortable on a 4-day trip. The hip belt pockets are big enough for a phone.

+
+ +
Free shipping on orders over $75! Use code TRAIL10 for 10% off.
+ + + diff --git a/crates/markkit-eval/data/pages/recipe.html b/crates/markkit-eval/data/pages/recipe.html new file mode 100644 index 0000000..d78ce8b --- /dev/null +++ b/crates/markkit-eval/data/pages/recipe.html @@ -0,0 +1,37 @@ + + +Weeknight Lemon Garlic Pasta - Pantry Table + +
+
+

Weeknight Lemon Garlic Pasta

+

Ready in 20 minutes with ingredients you probably already have.

+
Sponsored: Try MealBox free for 2 weeks
+ +

Ingredients

+ +

Instructions

+
    +
  1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water.
  2. +
  3. Warm the olive oil and garlic over low heat until fragrant, about 3 minutes.
  4. +
  5. Toss the pasta with the garlic oil, lemon zest, lemon juice and parmesan.
  6. +
  7. Loosen with pasta water as needed and serve immediately.
  8. +
+

Notes

+

For a dairy-free version, replace the parmesan with toasted breadcrumbs.

+
Sponsored: Chef's knife sale, 40% off
+

Rate this recipe:

+
+ + + diff --git a/crates/markkit-eval/data/pages/wiki_article.html b/crates/markkit-eval/data/pages/wiki_article.html new file mode 100644 index 0000000..7f10e80 --- /dev/null +++ b/crates/markkit-eval/data/pages/wiki_article.html @@ -0,0 +1,37 @@ + + +Lighthouse of Alexandria - OpenWiki + +
OpenWiki Log in Random article
+
+

Lighthouse of Alexandria

+
From OpenWiki, the free encyclopedia
+ + + + + +
Lighthouse of Alexandria
LocationPharos, Alexandria, Egypt
Builtc. 280 BC
Heightabout 100 m
+

The Lighthouse of Alexandria was a lighthouse built by the Ptolemaic Kingdom on the island of Pharos.[1] It was one of the Seven Wonders of the Ancient World.

+
+

Contents

+ +
+

Construction[edit]

+

Construction was completed during the reign of Ptolemy II Philadelphus. The architect was Sostratus of Cnidus.

+

Design

+

The tower had three stages: a square lower section, an octagonal middle section and a circular top.

+

Destruction[edit]

+

The lighthouse was badly damaged by earthquakes between 956 and 1323 AD and became an abandoned ruin.

+

References

+
    +
  1. Clayton, Peter; Price, Martin (1988). The Seven Wonders of the Ancient World. Routledge.
  2. +
+ +
+ + + diff --git a/crates/markkit-eval/results/markkit_extract-2026-10-27.json b/crates/markkit-eval/results/markkit_extract-2026-10-27.json new file mode 100644 index 0000000..b6d22d5 --- /dev/null +++ b/crates/markkit-eval/results/markkit_extract-2026-10-27.json @@ -0,0 +1,1317 @@ +{ + "cases": [ + { + "aggregate": 0.8254545454545456, + "eval": "markkit_extract", + "input": [ + "News article with cookie banner, site nav, inline ad, newsletter form, related-stories aside and footer" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "news_article", + "scores": [ + { + "pass": false, + "reason": "15/21 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 40%); failed: not_contains \"We use cookies\"; not_contains \"Refinance today\"; not_contains \"Advertisement\"; not_contains \"Related stories\"; not_contains \"New bike lanes\"; not_contains \"Get the morning briefing\"", + "scorer": "markkit_expectations", + "value": 0.7272727272727273 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "4/10 checks passed; failed: not_contains \"We use cookies\"; not_contains \"Refinance today\"; not_contains \"Advertisement\"; not_contains \"Related stories\"; not_contains \"New bike lanes\"; not_contains \"Get the morning briefing\"", + "scorer": "noise", + "value": 0.4 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "We use cookies to improve your experience. [Privacy policy](/privacy)\n\n# City Council Approves Riverside Park Expansion\n\nBy [Maria Chen](/authors/maria-chen) · March 14, 2026\n\n![Aerial view of Riverside Park](https://cdn.harborgazette.test/img/riverside-park.jpg)\n\n*The current park covers 42 acres along the east bank.*\n\nThe city council voted 7–2 on Tuesday to expand Riverside Park by **18 acres**, converting a former rail yard into wetlands and walking trails.\n\nAdvertisement[Refinance today at 2.9% APR](https://ads.example.test/click?id=991)\n\nThe project is expected to cost $14.2 million, with most of the funding coming from a state conservation grant approved last year.\n\n> This is the largest addition to our public green space in fifty years.\n\nConstruction will begin in the fall and is scheduled to finish by *spring 2028*. Residents can review the plans at the [city parks website](https://harbor.test/parks/riverside).\n\n## What happens to the rail yard\n\nContaminated soil will be removed in the first phase. The two historic signal towers will be preserved as observation decks.\n\n### Get the morning briefing\n\nRelated stories [New bike lanes open downtown](/local/bike-lanes) [Council debates 2027 budget](/local/budget)\n", + "iterations": 0, + "metrics": { + "extract_us": 2017.0, + "input_bytes": 2918.0, + "output_bytes": 1239.0, + "size_ratio": 0.42460589444825225 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 1.0, + "eval": "markkit_extract", + "input": [ + "News article with cookie banner, site nav, inline ad, newsletter form, related-stories aside and footer" + ], + "params": { + "mode": "main_content" + }, + "passed": true, + "sample": "news_article", + "scores": [ + { + "pass": true, + "reason": "21/21 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 100%)", + "scorer": "markkit_expectations", + "value": 1.0 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": true, + "reason": "10/10 checks passed", + "scorer": "noise", + "value": 1.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# City Council Approves Riverside Park Expansion\n\nBy [Maria Chen](/authors/maria-chen) · March 14, 2026\n\n![Aerial view of Riverside Park](https://cdn.harborgazette.test/img/riverside-park.jpg)\n\n*The current park covers 42 acres along the east bank.*\n\nThe city council voted 7–2 on Tuesday to expand Riverside Park by **18 acres**, converting a former rail yard into wetlands and walking trails.\n\nThe project is expected to cost $14.2 million, with most of the funding coming from a state conservation grant approved last year.\n\n> This is the largest addition to our public green space in fifty years.\n\nConstruction will begin in the fall and is scheduled to finish by *spring 2028*. Residents can review the plans at the [city parks website](https://harbor.test/parks/riverside).\n\n## What happens to the rail yard\n\nContaminated soil will be removed in the first phase. The two historic signal towers will be preserved as observation decks.\n", + "iterations": 0, + "metrics": { + "extract_us": 2355.0, + "input_bytes": 2918.0, + "output_bytes": 943.0, + "size_ratio": 0.3231665524331734 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.8476190476190476, + "eval": "markkit_extract", + "input": [ + "Docs page with sidebar nav, option table, TOML and Rust code blocks, callout, edit link and pager" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "docs_page", + "scores": [ + { + "pass": false, + "reason": "16/20 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 43%); failed: not_contains \"Quickstart\"; not_contains \"Edit this page\"; not_contains \"Last updated\"; not_contains \"Previous:\"", + "scorer": "markkit_expectations", + "value": 0.8095238095238095 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "3/7 checks passed; failed: not_contains \"Quickstart\"; not_contains \"Edit this page\"; not_contains \"Last updated\"; not_contains \"Previous:\"", + "scorer": "noise", + "value": 0.42857142857142855 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Configuration\n\nTideline reads its settings from `tideline.toml` in the project root. Every key is optional.\n\n## Options\n\n| Key | Type | Default | Description |\n| --- | --- | --- | --- |\n| `workers` | integer | 4 | Number of worker threads |\n| `timeout` | string | 30s | Request timeout |\n| `retry` | boolean | true | Retry failed requests once |\n\n## Example\n\n```toml\n[server]\nworkers = 8\ntimeout = \"45s\"\n\n[cache]\nenabled = true\n```\n\n## Loading from code\n\nUse `Config::load` to read the file manually:\n\n```rust\nuse tideline::Config;\n\nfn main() -> anyhow::Result<()> {\n let cfg = Config::load(\"tideline.toml\")?;\n println!(\"{} workers\", cfg.workers);\n Ok(())\n}\n```\n\n**Warning:** unknown keys are rejected at startup.\n\nSee [environment variables](/docs/env) to override settings per deployment.\n\n[Edit this page on GitHub](https://github.test/tideline/docs/edit/main/config.md) Last updated on Feb 2, 2026 [← Previous: Quickstart](/docs/quickstart) [Next: Environment →](/docs/env)\n", + "iterations": 0, + "metrics": { + "extract_us": 2829.0, + "input_bytes": 2223.0, + "output_bytes": 994.0, + "size_ratio": 0.4471434997750787 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.9238095238095236, + "eval": "markkit_extract", + "input": [ + "Docs page with sidebar nav, option table, TOML and Rust code blocks, callout, edit link and pager" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "docs_page", + "scores": [ + { + "pass": false, + "reason": "18/20 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 71%); failed: not_contains \"Edit this page\"; not_contains \"Last updated\"", + "scorer": "markkit_expectations", + "value": 0.9047619047619048 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "5/7 checks passed; failed: not_contains \"Edit this page\"; not_contains \"Last updated\"", + "scorer": "noise", + "value": 0.7142857142857143 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Configuration\n\nTideline reads its settings from `tideline.toml` in the project root. Every key is optional.\n\n## Options\n\n| Key | Type | Default | Description |\n| --- | --- | --- | --- |\n| `workers` | integer | 4 | Number of worker threads |\n| `timeout` | string | 30s | Request timeout |\n| `retry` | boolean | true | Retry failed requests once |\n\n## Example\n\n```toml\n[server]\nworkers = 8\ntimeout = \"45s\"\n\n[cache]\nenabled = true\n```\n\n## Loading from code\n\nUse `Config::load` to read the file manually:\n\n```rust\nuse tideline::Config;\n\nfn main() -> anyhow::Result<()> {\n let cfg = Config::load(\"tideline.toml\")?;\n println!(\"{} workers\", cfg.workers);\n Ok(())\n}\n```\n\n**Warning:** unknown keys are rejected at startup.\n\nSee [environment variables](/docs/env) to override settings per deployment.\n\n[Edit this page on GitHub](https://github.test/tideline/docs/edit/main/config.md) Last updated on Feb 2, 2026\n", + "iterations": 0, + "metrics": { + "extract_us": 5009.0, + "input_bytes": 2223.0, + "output_bytes": 913.0, + "size_ratio": 0.410706252811516 + }, + "timing": { + "duration_ms": 5 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.7176470588235294, + "eval": "markkit_extract", + "input": [ + "Engineering blog post with share buttons, author bio, comment section and newsletter popup" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "blog_post", + "scores": [ + { + "pass": false, + "reason": "9/16 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 0%); failed: not_contains \"We're hiring\"; not_contains \"Share on X\"; not_contains \"Share on LinkedIn\"; not_contains \"14 Comments\"; not_contains \"Did you consider Buck2\"; not_contains \"Log in to comment\"; not_contains \"Subscribe to our newsletter\"", + "scorer": "markkit_expectations", + "value": 0.5882352941176471 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "5/5 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "0/7 checks passed; failed: not_contains \"We're hiring\"; not_contains \"Share on X\"; not_contains \"Share on LinkedIn\"; not_contains \"14 Comments\"; not_contains \"Did you consider Buck2\"; not_contains \"Log in to comment\"; not_contains \"Subscribe to our newsletter\"", + "scorer": "noise", + "value": 0.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "[Lattice Engineering Blog](/) \\| [We're hiring\\!](/careers)\n\n# Why we moved our build to Bazel\n\nPosted by Jordan Alvarez on 9 January 2026 · 6 min read [Share on X](https://twitter.test/share?u=bazel) [Share on LinkedIn](https://linkedin.test/share?u=bazel)\n\nOur monorepo had grown to 1.4 million lines of code, and a clean CI build took **47 minutes**. Incremental builds were unreliable, so engineers ran clean builds out of habit.\n\n## What we tried first\n\n1. Caching `node_modules` between CI runs\n2. Splitting the test suite across 12 shards\n3. Upgrading CI runners to 32 cores\n\nEach change helped a little, but none fixed the root problem: the build had no accurate dependency graph.\n\n## Results\n\nAfter the migration, the median CI build takes 6 minutes, and 92% of builds hit the remote cache.\n\nThanks to the developer productivity team for leading the migration.\n\n![Jordan Alvarez](/avatars/jordan.png)\n\nJordan Alvarez is a staff engineer on the platform team.\n\n### 14 Comments\n\n**devops_dan** said: Great write-up! Did you consider Buck2? **sarahk** said: How long did the migration take? [Log in to comment](/login)\n\nSubscribe to our newsletter for weekly posts\\!\n", + "iterations": 0, + "metrics": { + "extract_us": 1477.0, + "input_bytes": 1878.0, + "output_bytes": 1175.0, + "size_ratio": 0.6256656017039404 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 1.0, + "eval": "markkit_extract", + "input": [ + "Engineering blog post with share buttons, author bio, comment section and newsletter popup" + ], + "params": { + "mode": "main_content" + }, + "passed": true, + "sample": "blog_post", + "scores": [ + { + "pass": true, + "reason": "16/16 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 100%)", + "scorer": "markkit_expectations", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "5/5 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": true, + "reason": "7/7 checks passed", + "scorer": "noise", + "value": 1.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Why we moved our build to Bazel\n\nPosted by Jordan Alvarez on 9 January 2026 · 6 min read\n\nOur monorepo had grown to 1.4 million lines of code, and a clean CI build took **47 minutes**. Incremental builds were unreliable, so engineers ran clean builds out of habit.\n\n## What we tried first\n\n1. Caching `node_modules` between CI runs\n2. Splitting the test suite across 12 shards\n3. Upgrading CI runners to 32 cores\n\nEach change helped a little, but none fixed the root problem: the build had no accurate dependency graph.\n\n## Results\n\nAfter the migration, the median CI build takes 6 minutes, and 92% of builds hit the remote cache.\n\nThanks to the developer productivity team for leading the migration.\n\n![Jordan Alvarez](/avatars/jordan.png)\n\nJordan Alvarez is a staff engineer on the platform team.\n", + "iterations": 0, + "metrics": { + "extract_us": 1772.0, + "input_bytes": 1878.0, + "output_bytes": 802.0, + "size_ratio": 0.42705005324813633 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.8388471177944862, + "eval": "markkit_extract", + "input": [ + "Product page with nav, breadcrumbs, buy box form, spec table, reviews, recommendations and promo banner" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "product_page", + "scores": [ + { + "pass": false, + "reason": "14/18 checks passed (completeness 83%, accuracy 100%, formatting 100%, noise 57%); failed: contains \"4.6 out of 5 stars\"; not_contains \"You may also like\"; not_contains \"Ridgeline 30L\"; not_contains \"TRAIL10\"", + "scorer": "markkit_expectations", + "value": 0.7894736842105263 + }, + { + "pass": false, + "reason": "5/6 checks passed; failed: contains \"4.6 out of 5 stars\"", + "scorer": "completeness", + "value": 0.8333333333333334 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "4/7 checks passed; failed: not_contains \"You may also like\"; not_contains \"Ridgeline 30L\"; not_contains \"TRAIL10\"", + "scorer": "noise", + "value": 0.5714285714285714 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "[Home](/) / [Packs](/packs) / Northwind Trail 40L\n\n# Northwind Trail 40L Backpack\n\n$189.00\n\nA lightweight pack for multi-day hikes, with a ventilated back panel and a removable rain cover.\n\n## Features\n\n- Recycled 210D ripstop nylon\n- Adjustable torso length (15–21 in)\n- Hydration sleeve fits a 3 L reservoir\n\n## Specifications\n\n| | |\n| --- | --- |\n| Weight | 1.2 kg |\n| Capacity | 40 L |\n| Dimensions | 64 × 32 × 24 cm |\n\n## Customer reviews\n\n4\\.6 out of 5 stars (212 reviews)\n\n**Comfortable on a 4-day trip.** The hip belt pockets are big enough for a phone.\n\nYou may also like [Ridgeline 30L Daypack](/p/ridgeline-30) [Carbon Trek Poles](/p/trek-poles)\n\nFree shipping on orders over $75! Use code TRAIL10 for 10% off.\n", + "iterations": 0, + "metrics": { + "extract_us": 2919.0, + "input_bytes": 1718.0, + "output_bytes": 728.0, + "size_ratio": 0.4237485448195577 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.956140350877193, + "eval": "markkit_extract", + "input": [ + "Product page with nav, breadcrumbs, buy box form, spec table, reviews, recommendations and promo banner" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "product_page", + "scores": [ + { + "pass": false, + "reason": "17/18 checks passed (completeness 83%, accuracy 100%, formatting 100%, noise 100%); failed: contains \"4.6 out of 5 stars\"", + "scorer": "markkit_expectations", + "value": 0.9473684210526316 + }, + { + "pass": false, + "reason": "5/6 checks passed; failed: contains \"4.6 out of 5 stars\"", + "scorer": "completeness", + "value": 0.8333333333333334 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": true, + "reason": "7/7 checks passed", + "scorer": "noise", + "value": 1.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Northwind Trail 40L Backpack\n\n$189.00\n\nA lightweight pack for multi-day hikes, with a ventilated back panel and a removable rain cover.\n\n## Features\n\n- Recycled 210D ripstop nylon\n- Adjustable torso length (15–21 in)\n- Hydration sleeve fits a 3 L reservoir\n\n## Specifications\n\n| | |\n| --- | --- |\n| Weight | 1.2 kg |\n| Capacity | 40 L |\n| Dimensions | 64 × 32 × 24 cm |\n\n## Customer reviews\n\n4\\.6 out of 5 stars (212 reviews)\n\n**Comfortable on a 4-day trip.** The hip belt pockets are big enough for a phone.\n", + "iterations": 0, + "metrics": { + "extract_us": 2001.0, + "input_bytes": 1718.0, + "output_bytes": 517.0, + "size_ratio": 0.3009313154831199 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.79, + "eval": "markkit_extract", + "input": [ + "Recipe with sponsored ad slots, ingredient list, numbered steps and a rating widget" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "recipe", + "scores": [ + { + "pass": false, + "reason": "11/15 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 20%); failed: not_contains \"Sponsored\"; not_contains \"MealBox\"; not_contains \"Chef's knife\"; not_contains \"Rate this recipe\"", + "scorer": "markkit_expectations", + "value": 0.75 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "7/7 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "1/5 checks passed; failed: not_contains \"Sponsored\"; not_contains \"MealBox\"; not_contains \"Chef's knife\"; not_contains \"Rate this recipe\"", + "scorer": "noise", + "value": 0.2 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Weeknight Lemon Garlic Pasta\n\nReady in 20 minutes with ingredients you probably already have.\n\nSponsored: Try MealBox free for 2 weeks\n\n- Prep: 5 min\n- Cook: 15 min\n- Serves: 4\n\n## Ingredients\n\n- 350 g spaghetti\n- 4 cloves garlic, thinly sliced\n- Zest and juice of 1 lemon\n- 60 ml olive oil\n- 40 g grated parmesan\n\n## Instructions\n\n1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water.\n2. Warm the olive oil and garlic over low heat until fragrant, about 3 minutes.\n3. Toss the pasta with the garlic oil, lemon zest, lemon juice and parmesan.\n4. Loosen with pasta water as needed and serve immediately.\n\n## Notes\n\nFor a dairy-free version, replace the parmesan with toasted breadcrumbs.\n\nSponsored: Chef's knife sale, 40% off\n\nRate this recipe:\n", + "iterations": 0, + "metrics": { + "extract_us": 2336.0, + "input_bytes": 1608.0, + "output_bytes": 777.0, + "size_ratio": 0.4832089552238806 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.9475, + "eval": "markkit_extract", + "input": [ + "Recipe with sponsored ad slots, ingredient list, numbered steps and a rating widget" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "recipe", + "scores": [ + { + "pass": false, + "reason": "14/15 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 80%); failed: not_contains \"Rate this recipe\"", + "scorer": "markkit_expectations", + "value": 0.9375 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "7/7 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "4/5 checks passed; failed: not_contains \"Rate this recipe\"", + "scorer": "noise", + "value": 0.8 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Weeknight Lemon Garlic Pasta\n\nReady in 20 minutes with ingredients you probably already have.\n\n- Prep: 5 min\n- Cook: 15 min\n- Serves: 4\n\n## Ingredients\n\n- 350 g spaghetti\n- 4 cloves garlic, thinly sliced\n- Zest and juice of 1 lemon\n- 60 ml olive oil\n- 40 g grated parmesan\n\n## Instructions\n\n1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water.\n2. Warm the olive oil and garlic over low heat until fragrant, about 3 minutes.\n3. Toss the pasta with the garlic oil, lemon zest, lemon juice and parmesan.\n4. Loosen with pasta water as needed and serve immediately.\n\n## Notes\n\nFor a dairy-free version, replace the parmesan with toasted breadcrumbs.\n\nRate this recipe:\n", + "iterations": 0, + "metrics": { + "extract_us": 1564.0, + "input_bytes": 1608.0, + "output_bytes": 697.0, + "size_ratio": 0.43345771144278605 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.62, + "eval": "markkit_extract", + "input": [ + "Encyclopedia article with infobox, inline citations, table of contents, edit links, categories and footer" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "wiki_article", + "scores": [ + { + "pass": false, + "reason": "8/17 checks passed (completeness 100%, accuracy 100%, formatting 60%, noise 0%); failed: heading \"## Construction\"; heading \"## Destruction\"; not_contains \"Log in\"; not_contains \"Random article\"; not_contains \"[edit]\"; not_contains \"From OpenWiki, the free encyclopedia\"; not_contains \"Contents\"; not_contains \"Categories:\"; not_contains \"last edited on\"", + "scorer": "markkit_expectations", + "value": 0.5 + }, + { + "pass": true, + "reason": "5/5 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": false, + "reason": "2/4 checks passed; failed: heading \"## Construction\"; heading \"## Destruction\"", + "scorer": "formatting", + "value": 0.6 + }, + { + "pass": false, + "reason": "0/7 checks passed; failed: not_contains \"Log in\"; not_contains \"Random article\"; not_contains \"[edit]\"; not_contains \"From OpenWiki, the free encyclopedia\"; not_contains \"Contents\"; not_contains \"Categories:\"; not_contains \"last edited on\"", + "scorer": "noise", + "value": 0.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "[OpenWiki](/wiki/Main_Page) [Log in](/login) [Random article](/random)\n\n# Lighthouse of Alexandria\n\nFrom OpenWiki, the free encyclopedia\n\n| Lighthouse of Alexandria | |\n| --- | --- |\n| Location | Pharos, Alexandria, Egypt |\n| Built | c. 280 BC |\n| Height | about 100 m |\n\nThe **Lighthouse of Alexandria** was a lighthouse built by the Ptolemaic Kingdom on the island of Pharos.[\\[1\\]](#cite_note-1) It was one of the Seven Wonders of the Ancient World.\n\n## Contents\n\n- [1 Construction](#Construction)\n- [2 Destruction](#Destruction)\n\n## Construction \\[[edit](/edit?section=1)\\]\n\nConstruction was completed during the reign of Ptolemy II Philadelphus. The architect was Sostratus of Cnidus.\n\n### Design\n\nThe tower had three stages: a square lower section, an octagonal middle section and a circular top.\n\n## Destruction \\[[edit](/edit?section=2)\\]\n\nThe lighthouse was badly damaged by earthquakes between 956 and 1323 AD and became an abandoned ruin.\n\n## References\n\n1. Clayton, Peter; Price, Martin (1988). *The Seven Wonders of the Ancient World*. Routledge.\n\nCategories: [Lighthouses](/wiki/Category:Lighthouses) \\| [Seven Wonders of the Ancient World](/wiki/Category:Seven_Wonders) This page was last edited on 3 March 2026. Text is available under the Creative Commons Attribution-ShareAlike License.\n", + "iterations": 0, + "metrics": { + "extract_us": 1856.0, + "input_bytes": 2171.0, + "output_bytes": 1307.0, + "size_ratio": 0.6020267157991709 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.739047619047619, + "eval": "markkit_extract", + "input": [ + "Encyclopedia article with infobox, inline citations, table of contents, edit links, categories and footer" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "wiki_article", + "scores": [ + { + "pass": false, + "reason": "11/17 checks passed (completeness 100%, accuracy 100%, formatting 60%, noise 43%); failed: heading \"## Construction\"; heading \"## Destruction\"; not_contains \"[edit]\"; not_contains \"From OpenWiki, the free encyclopedia\"; not_contains \"Contents\"; not_contains \"Categories:\"", + "scorer": "markkit_expectations", + "value": 0.6666666666666666 + }, + { + "pass": true, + "reason": "5/5 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "1/1 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": false, + "reason": "2/4 checks passed; failed: heading \"## Construction\"; heading \"## Destruction\"", + "scorer": "formatting", + "value": 0.6 + }, + { + "pass": false, + "reason": "3/7 checks passed; failed: not_contains \"[edit]\"; not_contains \"From OpenWiki, the free encyclopedia\"; not_contains \"Contents\"; not_contains \"Categories:\"", + "scorer": "noise", + "value": 0.42857142857142855 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Lighthouse of Alexandria\n\nFrom OpenWiki, the free encyclopedia\n\n| Lighthouse of Alexandria | |\n| --- | --- |\n| Location | Pharos, Alexandria, Egypt |\n| Built | c. 280 BC |\n| Height | about 100 m |\n\nThe **Lighthouse of Alexandria** was a lighthouse built by the Ptolemaic Kingdom on the island of Pharos.[\\[1\\]](#cite_note-1) It was one of the Seven Wonders of the Ancient World.\n\n## Contents\n\n- [1 Construction](#Construction)\n- [2 Destruction](#Destruction)\n\n## Construction \\[[edit](/edit?section=1)\\]\n\nConstruction was completed during the reign of Ptolemy II Philadelphus. The architect was Sostratus of Cnidus.\n\n### Design\n\nThe tower had three stages: a square lower section, an octagonal middle section and a circular top.\n\n## Destruction \\[[edit](/edit?section=2)\\]\n\nThe lighthouse was badly damaged by earthquakes between 956 and 1323 AD and became an abandoned ruin.\n\n## References\n\n1. Clayton, Peter; Price, Martin (1988). *The Seven Wonders of the Ancient World*. Routledge.\n\nCategories: [Lighthouses](/wiki/Category:Lighthouses) \\| [Seven Wonders of the Ancient World](/wiki/Category:Seven_Wonders)\n", + "iterations": 0, + "metrics": { + "extract_us": 2371.0, + "input_bytes": 2171.0, + "output_bytes": 1114.0, + "size_ratio": 0.5131275909719023 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.8625, + "eval": "markkit_extract", + "input": [ + "Q&A thread with vote widgets, tags, accepted answer with code, user cards and sidebar" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "forum_thread", + "scores": [ + { + "pass": false, + "reason": "12/15 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 50%); failed: not_contains \"Viewed 48k times\"; not_contains \"Hot Network Questions\"; not_contains \"CSS grid\"", + "scorer": "markkit_expectations", + "value": 0.8125 + }, + { + "pass": true, + "reason": "5/5 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "3/6 checks passed; failed: not_contains \"Viewed 48k times\"; not_contains \"Hot Network Questions\"; not_contains \"CSS grid\"", + "scorer": "noise", + "value": 0.5 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# How do I reverse a linked list in place?\n\nAsked 3 years ago · Viewed 48k times 312\n\nI have a singly linked list and want to reverse it without allocating a new list. My attempt loses the rest of the list after the first node:\n\n```\nwhile node:\n node.next = prev\n prev = node\n node = node.next\n```\n\n[python](/tags/python) [linked-list](/tags/linked-list)\n\n## 2 Answers\n\n518 Accepted\n\nYou overwrite `node.next` before saving it. Keep a reference to the next node first:\n\n```python\ndef reverse(head):\n prev = None\n node = head\n while node:\n nxt = node.next\n node.next = prev\n prev = node\n node = nxt\n return prev\n```\n\nThis runs in O(n) time and O(1) extra space.\n\nanswered by [ada](/users/88/ada) 41\n\nA recursive version also works, but it uses O(n) stack space and can hit the recursion limit on long lists.\n\n### Hot Network Questions\n\n- [Why is my CSS grid collapsing?](/q/1)\n", + "iterations": 0, + "metrics": { + "extract_us": 1587.0, + "input_bytes": 2048.0, + "output_bytes": 928.0, + "size_ratio": 0.453125 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.5116666666666667, + "eval": "markkit_extract", + "input": [ + "Q&A thread with vote widgets, tags, accepted answer with code, user cards and sidebar" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "forum_thread", + "scores": [ + { + "pass": false, + "reason": "10/15 checks passed (completeness 60%, accuracy 0%, formatting 33%, noise 100%); failed: heading \"# How do I reverse a linked list in place?\"; contains \"reverse it without allocating a new list\"; contains \"recursive version also works\"; order \"without allocating a new list\" before \"You overwrite\"; order \"O(1) extra space\" before \"recursive version\"", + "scorer": "markkit_expectations", + "value": 0.625 + }, + { + "pass": false, + "reason": "3/5 checks passed; failed: contains \"reverse it without allocating a new list\"; contains \"recursive version also works\"", + "scorer": "completeness", + "value": 0.6 + }, + { + "pass": false, + "reason": "0/2 checks passed; failed: order \"without allocating a new list\" before \"You overwrite\"; order \"O(1) extra space\" before \"recursive version\"", + "scorer": "accuracy", + "value": 0.0 + }, + { + "pass": false, + "reason": "1/2 checks passed; failed: heading \"# How do I reverse a linked list in place?\"", + "scorer": "formatting", + "value": 0.3333333333333333 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "noise", + "value": 1.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "You overwrite `node.next` before saving it. Keep a reference to the next node first:\n\n```python\ndef reverse(head):\n prev = None\n node = head\n while node:\n nxt = node.next\n node.next = prev\n prev = node\n node = nxt\n return prev\n```\n\nThis runs in O(n) time and O(1) extra space.\n", + "iterations": 0, + "metrics": { + "extract_us": 1744.0, + "input_bytes": 2048.0, + "output_bytes": 317.0, + "size_ratio": 0.15478515625 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.9200000000000002, + "eval": "markkit_extract", + "input": [ + "Release notes with nested lists, strikethrough, issue links and call-to-action box" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "changelog", + "scores": [ + { + "pass": false, + "reason": "13/14 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 67%); failed: not_contains \"Start your free trial\"", + "scorer": "markkit_expectations", + "value": 0.9333333333333332 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "2/3 checks passed; failed: not_contains \"Start your free trial\"", + "scorer": "noise", + "value": 0.6666666666666666 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Changelog\n\nAll notable changes to Quillstack are documented here. Subscribe via [RSS](/changelog.rss).\n\n## v2.4.0 — 2026-04-02\n\n### Added\n\n- Offline mode for the desktop editor\n- Export to EPUB\n - Custom cover images\n - Table of contents generation\n\n### Fixed\n\n- Crash when pasting images larger than 20 MB ([#1182](https://git.test/quillstack/issues/1182))\n- ~~Spellcheck ignored code blocks~~ Spellcheck now skips code blocks\n\n## v2.3.1 — 2026-03-10\n\n### Security\n\n- Sanitize SVG uploads to prevent stored XSS\n\nReady to write? [Start your free trial](/signup)\n", + "iterations": 0, + "metrics": { + "extract_us": 1198.0, + "input_bytes": 1317.0, + "output_bytes": 570.0, + "size_ratio": 0.4328018223234624 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.9200000000000002, + "eval": "markkit_extract", + "input": [ + "Release notes with nested lists, strikethrough, issue links and call-to-action box" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "changelog", + "scores": [ + { + "pass": false, + "reason": "13/14 checks passed (completeness 100%, accuracy 100%, formatting 100%, noise 67%); failed: not_contains \"Start your free trial\"", + "scorer": "markkit_expectations", + "value": 0.9333333333333332 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "2/3 checks passed; failed: not_contains \"Start your free trial\"", + "scorer": "noise", + "value": 0.6666666666666666 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Changelog\n\nAll notable changes to Quillstack are documented here. Subscribe via [RSS](/changelog.rss).\n\n## v2.4.0 — 2026-04-02\n\n### Added\n\n- Offline mode for the desktop editor\n- Export to EPUB\n - Custom cover images\n - Table of contents generation\n\n### Fixed\n\n- Crash when pasting images larger than 20 MB ([#1182](https://git.test/quillstack/issues/1182))\n- ~~Spellcheck ignored code blocks~~ Spellcheck now skips code blocks\n\n## v2.3.1 — 2026-03-10\n\n### Security\n\n- Sanitize SVG uploads to prevent stored XSS\n\nReady to write? [Start your free trial](/signup)\n", + "iterations": 0, + "metrics": { + "extract_us": 1425.0, + "input_bytes": 1317.0, + "output_bytes": 570.0, + "size_ratio": 0.4328018223234624 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.7633333333333333, + "eval": "markkit_extract", + "input": [ + "Data report with a captioned table including thead, tbody and tfoot" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "data_table", + "scores": [ + { + "pass": false, + "reason": "10/14 checks passed (completeness 40%, accuracy 100%, formatting 100%, noise 67%); failed: contains \"Total\"; contains \"72.4\"; contains \"Quarterly generation\"; not_contains \"Share:\"", + "scorer": "markkit_expectations", + "value": 0.75 + }, + { + "pass": false, + "reason": "2/5 checks passed; failed: contains \"Total\"; contains \"72.4\"; contains \"Quarterly generation\"", + "scorer": "completeness", + "value": 0.4 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": false, + "reason": "2/3 checks passed; failed: not_contains \"Share:\"", + "scorer": "noise", + "value": 0.6666666666666666 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Electricity Generation by Source, 2025\n\nFigures are in terawatt-hours (TWh). Source: national grid operator, published February 2026.\n\n| Source | Q1 | Q2 | Q3 | Q4 |\n| --- | --- | --- | --- | --- |\n| Wind | 21.4 | 15.2 | 12.8 | 24.9 |\n| Solar | 4.1 | 11.7 | 13.3 | 3.6 |\n| Natural gas | 30.2 | 22.5 | 26.0 | 31.8 |\n| Nuclear | 12.0 | 11.6 | 11.9 | 12.1 |\n\nWind overtook natural gas in no quarter, but the gap narrowed to **6.9 TWh** in Q4.\n\n[Download the data (CSV)](/reports/2025-energy.csv)\n\nShare: [X](https://twitter.test/s) [Email](https://mail.test/s)\n", + "iterations": 0, + "metrics": { + "extract_us": 2314.0, + "input_bytes": 1397.0, + "output_bytes": 560.0, + "size_ratio": 0.4008589835361489 + }, + "timing": { + "duration_ms": 2 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.8425, + "eval": "markkit_extract", + "input": [ + "Data report with a captioned table including thead, tbody and tfoot" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "data_table", + "scores": [ + { + "pass": false, + "reason": "11/14 checks passed (completeness 40%, accuracy 100%, formatting 100%, noise 100%); failed: contains \"Total\"; contains \"72.4\"; contains \"Quarterly generation\"", + "scorer": "markkit_expectations", + "value": 0.8125 + }, + { + "pass": false, + "reason": "2/5 checks passed; failed: contains \"Total\"; contains \"72.4\"; contains \"Quarterly generation\"", + "scorer": "completeness", + "value": 0.4 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "pass": true, + "reason": "4/4 checks passed", + "scorer": "formatting", + "value": 1.0 + }, + { + "pass": true, + "reason": "3/3 checks passed", + "scorer": "noise", + "value": 1.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "# Electricity Generation by Source, 2025\n\nFigures are in terawatt-hours (TWh). Source: national grid operator, published February 2026.\n\n| Source | Q1 | Q2 | Q3 | Q4 |\n| --- | --- | --- | --- | --- |\n| Wind | 21.4 | 15.2 | 12.8 | 24.9 |\n| Solar | 4.1 | 11.7 | 13.3 | 3.6 |\n| Natural gas | 30.2 | 22.5 | 26.0 | 31.8 |\n| Nuclear | 12.0 | 11.6 | 11.9 | 12.1 |\n\nWind overtook natural gas in no quarter, but the gap narrowed to **6.9 TWh** in Q4.\n\n[Download the data (CSV)](/reports/2025-energy.csv)\n", + "iterations": 0, + "metrics": { + "extract_us": 1746.0, + "input_bytes": 1397.0, + "output_bytes": 495.0, + "size_ratio": 0.3543307086614173 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.6607142857142857, + "eval": "markkit_extract", + "input": [ + "1990s table-layout page with font tags, unclosed paragraphs, line breaks and a hit counter" + ], + "params": { + "mode": "default" + }, + "passed": false, + "sample": "legacy_layout", + "scores": [ + { + "pass": false, + "reason": "8/12 checks passed (completeness 100%, accuracy 100%, noise 0%); failed: not_contains \"MENU\"; not_contains \"Guestbook\"; not_contains \"visitor number\"; not_contains \"Netscape Navigator\"", + "scorer": "markkit_expectations", + "value": 0.6428571428571429 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "na": true, + "pass": false, + "reason": "no checks for this metric", + "scorer": "formatting", + "value": 0.0 + }, + { + "pass": false, + "reason": "0/4 checks passed; failed: not_contains \"MENU\"; not_contains \"Guestbook\"; not_contains \"visitor number\"; not_contains \"Netscape Navigator\"", + "scorer": "noise", + "value": 0.0 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "| | |\n| --- | --- |\n| **MENU** [Home](index.htm) [Our Layout](layout.htm) [Join](join.htm) [Guestbook](guestbook.htm) | **Pine Hollow Model Railroad Club** Welcome to the home of the Pine Hollow Model Railroad Club! We meet every **Tuesday at 7 PM** in the basement of the Grange Hall, 18 Mill Road. Our HO scale layout has over 600 feet of track and models the Boston & Maine railroad in 1952. New members are always welcome. Annual dues are $40. Contact our secretary, Walt Brennan, at [walt@pinehollow.test](mailto:walt@pinehollow.test). You are visitor number ![004127](counter.gif) since 1998. Best viewed in Netscape Navigator 4.0 at 800x600. |\n", + "iterations": 0, + "metrics": { + "extract_us": 1322.0, + "input_bytes": 1283.0, + "output_bytes": 661.0, + "size_ratio": 0.5151987529228371 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + }, + { + "aggregate": 0.8642857142857143, + "eval": "markkit_extract", + "input": [ + "1990s table-layout page with font tags, unclosed paragraphs, line breaks and a hit counter" + ], + "params": { + "mode": "main_content" + }, + "passed": false, + "sample": "legacy_layout", + "scores": [ + { + "pass": false, + "reason": "10/12 checks passed (completeness 100%, accuracy 100%, noise 60%); failed: not_contains \"visitor number\"; not_contains \"Netscape Navigator\"", + "scorer": "markkit_expectations", + "value": 0.8571428571428571 + }, + { + "pass": true, + "reason": "6/6 checks passed", + "scorer": "completeness", + "value": 1.0 + }, + { + "pass": true, + "reason": "2/2 checks passed", + "scorer": "accuracy", + "value": 1.0 + }, + { + "na": true, + "pass": false, + "reason": "no checks for this metric", + "scorer": "formatting", + "value": 0.0 + }, + { + "pass": false, + "reason": "2/4 checks passed; failed: not_contains \"visitor number\"; not_contains \"Netscape Navigator\"", + "scorer": "noise", + "value": 0.6 + } + ], + "skipped": false, + "target": "markkit", + "transcript": { + "final_response": "**Pine Hollow Model Railroad Club**\n\nWelcome to the home of the Pine Hollow Model Railroad Club! We meet every **Tuesday at 7 PM** in the basement of the Grange Hall, 18 Mill Road.\n\nOur HO scale layout has over 600 feet of track and models the Boston & Maine railroad in 1952.\n\nNew members are always welcome. \n Annual dues are $40. \n Contact our secretary, Walt Brennan, at [walt@pinehollow.test](mailto:walt@pinehollow.test).\n\n---\n\nYou are visitor number ![004127](counter.gif) since 1998. Best viewed in Netscape Navigator 4.0 at 800x600.\n", + "iterations": 0, + "metrics": { + "extract_us": 1488.0, + "input_bytes": 1283.0, + "output_bytes": 544.0, + "size_ratio": 0.4240062353858145 + }, + "timing": { + "duration_ms": 1 + }, + "tool_calls_count": 0, + "usage": { + "cost_usd": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + ], + "summary": { + "failed": 18, + "na": 0, + "passed": 2, + "scored": 20, + "skipped": 0, + "total_cost_usd": 0.0, + "total_duration_ms": 32, + "total_tokens": 0, + "total_tool_calls": 0 + } +} \ No newline at end of file diff --git a/crates/markkit-eval/results/markkit_extract-2026-10-27.md b/crates/markkit-eval/results/markkit_extract-2026-10-27.md new file mode 100644 index 0000000..5c7d1e1 --- /dev/null +++ b/crates/markkit-eval/results/markkit_extract-2026-10-27.md @@ -0,0 +1,48 @@ +# markkit_extract results, 2026-10-27 + +Run: `just eval` (mira 0.5, `markkit_extract`, both `mode` values) on main at 5a7434c (IR v2, limits, main-content detection, markdown escaping #23, text input). Raw mira report: [`markkit_extract-2026-10-27.json`](markkit_extract-2026-10-27.json). Design: [knowledge/operations/eval.md](../../../knowledge/operations/eval.md). + +## Summary + +| mode | pages passing | overall | completeness | accuracy | formatting | noise | +|---|---|---|---|---|---|---| +| `default` | 0/10 | 73% | 92% | 100% | 96% | 34% | +| `main_content` | 2/10 | 87% | 88% | 90% | 88% | 82% | + +Overall = mean weighted check pass rate; a page passes only when every check passes. + +## Per page (weighted rate) + +| page | default | main_content | +|---|---|---| +| news_article | 73% | 100% | +| docs_page | 81% | 90% | +| blog_post | 59% | 100% | +| product_page | 79% | 95% | +| recipe | 75% | 94% | +| wiki_article | 50% | 67% | +| forum_thread | 81% | 62% | +| changelog | 93% | 93% | +| data_table | 75% | 81% | +| legacy_layout | 64% | 86% | + +## Failed checks + +- **news_article** `default`: not_contains "We use cookies"; not_contains "Refinance today"; not_contains "Advertisement"; not_contains "Related stories"; not_contains "New bike lanes"; not_contains "Get the morning briefing" +- **docs_page** `default`: not_contains "Quickstart"; not_contains "Edit this page"; not_contains "Last updated"; not_contains "Previous:" +- **docs_page** `main_content`: not_contains "Edit this page"; not_contains "Last updated" +- **blog_post** `default`: not_contains "We're hiring"; not_contains "Share on X"; not_contains "Share on LinkedIn"; not_contains "14 Comments"; not_contains "Did you consider Buck2"; not_contains "Log in to comment"; not_contains "Subscribe to our newsletter" +- **product_page** `default`: contains "4.6 out of 5 stars"; not_contains "You may also like"; not_contains "Ridgeline 30L"; not_contains "TRAIL10" +- **product_page** `main_content`: contains "4.6 out of 5 stars" +- **recipe** `default`: not_contains "Sponsored"; not_contains "MealBox"; not_contains "Chef's knife"; not_contains "Rate this recipe" +- **recipe** `main_content`: not_contains "Rate this recipe" +- **wiki_article** `default`: heading "## Construction"; heading "## Destruction"; not_contains "Log in"; not_contains "Random article"; not_contains "[edit]"; not_contains "From OpenWiki, the free encyclopedia"; not_contains "Contents"; not_contains "Categories:"; not_contains "last edited on" +- **wiki_article** `main_content`: heading "## Construction"; heading "## Destruction"; not_contains "[edit]"; not_contains "From OpenWiki, the free encyclopedia"; not_contains "Contents"; not_contains "Categories:" +- **forum_thread** `default`: not_contains "Viewed 48k times"; not_contains "Hot Network Questions"; not_contains "CSS grid" +- **forum_thread** `main_content`: heading "# How do I reverse a linked list in place?"; contains "reverse it without allocating a new list"; contains "recursive version also works"; order "without allocating a new list" before "You overwrite"; order "O(1) extra space" before "recursive version" +- **changelog** `default`: not_contains "Start your free trial" +- **changelog** `main_content`: not_contains "Start your free trial" +- **data_table** `default`: contains "Total"; contains "72.4"; contains "Quarterly generation"; not_contains "Share:" +- **data_table** `main_content`: contains "Total"; contains "72.4"; contains "Quarterly generation" +- **legacy_layout** `default`: not_contains "MENU"; not_contains "Guestbook"; not_contains "visitor number"; not_contains "Netscape Navigator" +- **legacy_layout** `main_content`: not_contains "visitor number"; not_contains "Netscape Navigator" diff --git a/crates/markkit-eval/src/checks.rs b/crates/markkit-eval/src/checks.rs new file mode 100644 index 0000000..fbd69d5 --- /dev/null +++ b/crates/markkit-eval/src/checks.rs @@ -0,0 +1,223 @@ +// Deterministic expectation checks over extracted markdown. +// +// Pure string logic, no regex: page text is data, never a pattern. Each check +// returns pass/fail plus a short detail for the report. Per-metric aggregation +// (weighted pass rate) lives here too so scorers and tests share it. + +use std::collections::BTreeMap; + +use crate::dataset::{Check, Expectation, Metric}; + +/// Outcome of one expectation. +#[derive(Debug, Clone)] +pub struct CheckResult { + pub metric: Metric, + pub weight: f64, + pub passed: bool, + pub detail: String, +} + +/// Run one check against `markdown`. +pub fn run_check(check: &Check, markdown: &str) -> (bool, String) { + match check { + Check::Contains { text } => ( + markdown.contains(text.as_str()), + format!("contains {text:?}"), + ), + Check::NotContains { text } => ( + !markdown.contains(text.as_str()), + format!("not_contains {text:?}"), + ), + Check::Line { text } => ( + markdown.lines().any(|l| l.trim_end() == text), + format!("line {text:?}"), + ), + Check::Heading { level, text } => { + let want = format!("{} {}", "#".repeat(usize::from(*level)), text); + ( + (1..=6).contains(level) && markdown.lines().any(|l| l.trim_end() == want), + format!("heading {want:?}"), + ) + } + Check::Link { text, url } => { + let want = format!("[{text}]({url})"); + (markdown.contains(&want), format!("link {want:?}")) + } + Check::Order { first, then } => { + let ok = match (markdown.find(first.as_str()), markdown.find(then.as_str())) { + (Some(a), Some(b)) => a < b, + _ => false, + }; + (ok, format!("order {first:?} before {then:?}")) + } + } +} + +/// Run every expectation. +pub fn evaluate(expectations: &[Expectation], markdown: &str) -> Vec { + expectations + .iter() + .map(|e| { + let (passed, detail) = run_check(&e.check, markdown); + CheckResult { + metric: e.metric(), + weight: e.weight, + passed, + detail, + } + }) + .collect() +} + +/// Weighted pass rate of `results`; `None` when there is nothing to score. +pub fn weighted_rate<'a>(results: impl IntoIterator) -> Option { + let (mut total, mut passed) = (0.0, 0.0); + for r in results { + total += r.weight; + if r.passed { + passed += r.weight; + } + } + (total > 0.0).then(|| passed / total) +} + +/// Weighted pass rate per metric (metrics without checks are absent). +pub fn per_metric(results: &[CheckResult]) -> BTreeMap { + Metric::ALL + .into_iter() + .filter_map(|m| weighted_rate(results.iter().filter(|r| r.metric == m)).map(|v| (m, v))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const MD: &str = + "# Title\n\nIntro with [docs](https://x.test/docs).\n\n## Usage \n\n- one\n- two\n"; + + fn c(check: Check) -> bool { + run_check(&check, MD).0 + } + + fn s(v: &str) -> String { + v.to_string() + } + + #[test] + fn contains_and_not_contains() { + assert!(c(Check::Contains { + text: s("Intro with") + })); + assert!(!c(Check::Contains { text: s("missing") })); + assert!(c(Check::NotContains { text: s("Cookie") })); + assert!(!c(Check::NotContains { text: s("Title") })); + } + + #[test] + fn line_matches_whole_line_ignoring_trailing_space() { + assert!(c(Check::Line { text: s("- one") })); + assert!(c(Check::Line { + text: s("## Usage") + })); + assert!(!c(Check::Line { text: s("one") })); + } + + #[test] + fn heading_level_and_text_exact() { + assert!(c(Check::Heading { + level: 1, + text: s("Title") + })); + assert!(c(Check::Heading { + level: 2, + text: s("Usage") + })); + assert!(!c(Check::Heading { + level: 2, + text: s("Title") + })); + assert!(!c(Check::Heading { + level: 1, + text: s("Tit") + })); + // Out-of-range levels never pass, even on a matching-looking line. + assert!( + !run_check( + &Check::Heading { + level: 0, + text: s("x") + }, + " x" + ) + .0 + ); + assert!( + !run_check( + &Check::Heading { + level: 7, + text: s("x") + }, + "####### x" + ) + .0 + ); + } + + #[test] + fn link_needs_exact_text_and_url() { + assert!(c(Check::Link { + text: s("docs"), + url: s("https://x.test/docs") + })); + assert!(!c(Check::Link { + text: s("docs"), + url: s("https://x.test/") + })); + assert!(!c(Check::Link { + text: s("Docs"), + url: s("https://x.test/docs") + })); + } + + #[test] + fn order_requires_both_and_sequence() { + assert!(c(Check::Order { + first: s("Title"), + then: s("Usage") + })); + assert!(!c(Check::Order { + first: s("Usage"), + then: s("Title") + })); + assert!(!c(Check::Order { + first: s("Title"), + then: s("absent") + })); + assert!(!c(Check::Order { + first: s("absent"), + then: s("Title") + })); + } + + #[test] + fn weighted_rates_per_metric() { + let r = |metric, weight, passed| CheckResult { + metric, + weight, + passed, + detail: String::new(), + }; + let results = vec![ + r(Metric::Completeness, 3.0, true), + r(Metric::Completeness, 1.0, false), + r(Metric::Noise, 1.0, false), + ]; + assert_eq!(weighted_rate(&results), Some(0.6)); + let m = per_metric(&results); + assert_eq!(m[&Metric::Completeness], 0.75); + assert_eq!(m[&Metric::Noise], 0.0); + assert!(!m.contains_key(&Metric::Formatting)); + assert_eq!(weighted_rate(&[]), None); + } +} diff --git a/crates/markkit-eval/src/dataset.rs b/crates/markkit-eval/src/dataset.rs new file mode 100644 index 0000000..b346fb3 --- /dev/null +++ b/crates/markkit-eval/src/dataset.rs @@ -0,0 +1,245 @@ +// Eval dataset: extraction tasks (JSONL) and the pages they extract. +// +// Decisions: +// - Pages live as real `.html` files under `data/pages/` (reviewable, open in a +// browser) and are embedded with `include_str!` via `PAGES`, so the study has +// no runtime path dependence. A test keeps `PAGES` and the directory in sync. +// - Expectations are typed (`check` tag) instead of bashkit's `kind:arg` +// strings: page text routinely contains `:`, which made string parsing +// ambiguous for two-argument checks (`link`, `order`). +// - Each expectation belongs to one quality metric (completeness, accuracy, +// formatting, noise); the default follows the check kind. Scorers aggregate +// per metric. See knowledge/operations/eval.md. + +use serde::{Deserialize, Serialize}; + +/// One extraction task: a page plus what ideal markdown for it must satisfy. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExtractTask { + pub id: String, + pub category: String, + pub description: String, + /// Page file name under `data/pages/` (a key of [`PAGES`]). + pub page: String, + pub expectations: Vec, +} + +/// Quality dimension an expectation measures. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Metric { + /// Important content is preserved. + Completeness, + /// Content is faithful: right text, right order, right targets. + Accuracy, + /// Markdown structure is right (headings, lists, tables, code fences). + Formatting, + /// Non-content (nav, ads, banners, comments) is excluded. + Noise, +} + +impl Metric { + pub const ALL: [Metric; 4] = [ + Metric::Completeness, + Metric::Accuracy, + Metric::Formatting, + Metric::Noise, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Metric::Completeness => "completeness", + Metric::Accuracy => "accuracy", + Metric::Formatting => "formatting", + Metric::Noise => "noise", + } + } +} + +/// A deterministic check against the extracted markdown. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "check", rename_all = "snake_case")] +pub enum Check { + /// `text` occurs somewhere in the output. + Contains { text: String }, + /// `text` does not occur anywhere in the output. + NotContains { text: String }, + /// Some output line equals `text` (trailing whitespace ignored). + Line { text: String }, + /// An ATX heading of exactly `level` with exactly `text`. + Heading { level: u8, text: String }, + /// An inline link `[text](url)`. + Link { text: String, url: String }, + /// Both occur and the first occurrence of `first` precedes `then`. + Order { first: String, then: String }, +} + +impl Check { + /// Metric a check measures when the task does not say. + pub fn default_metric(&self) -> Metric { + match self { + Check::Contains { .. } => Metric::Completeness, + Check::NotContains { .. } => Metric::Noise, + Check::Line { .. } | Check::Heading { .. } => Metric::Formatting, + Check::Link { .. } | Check::Order { .. } => Metric::Accuracy, + } + } +} + +/// A check with its metric and weight. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Expectation { + #[serde(flatten)] + pub check: Check, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metric: Option, + #[serde(default = "default_weight")] + pub weight: f64, +} + +fn default_weight() -> f64 { + 1.0 +} + +impl Expectation { + pub fn metric(&self) -> Metric { + self.metric.unwrap_or_else(|| self.check.default_metric()) + } +} + +/// Embedded task list. +pub const TASKS: &str = include_str!("../data/extract-tasks.jsonl"); + +/// Embedded pages, keyed by file name under `data/pages/`. +pub const PAGES: &[(&str, &str)] = &[ + ( + "changelog.html", + include_str!("../data/pages/changelog.html"), + ), + ( + "data_table.html", + include_str!("../data/pages/data_table.html"), + ), + ( + "docs_page.html", + include_str!("../data/pages/docs_page.html"), + ), + ( + "forum_thread.html", + include_str!("../data/pages/forum_thread.html"), + ), + ( + "legacy_layout.html", + include_str!("../data/pages/legacy_layout.html"), + ), + ( + "news_article.html", + include_str!("../data/pages/news_article.html"), + ), + ( + "product_page.html", + include_str!("../data/pages/product_page.html"), + ), + ("recipe.html", include_str!("../data/pages/recipe.html")), + ( + "blog_post.html", + include_str!("../data/pages/blog_post.html"), + ), + ( + "wiki_article.html", + include_str!("../data/pages/wiki_article.html"), + ), +]; + +/// Look up an embedded page by file name. +pub fn page(name: &str) -> Option<&'static str> { + PAGES + .iter() + .find(|(n, _)| *n == name) + .map(|(_, html)| *html) +} + +/// Parse JSONL tasks. Blank and `#` / `//` comment lines are skipped; a +/// malformed line is an error (a silently dropped task would skew scores). +pub fn parse_tasks(text: &str) -> Result, String> { + text.lines() + .enumerate() + .map(|(i, l)| (i + 1, l.trim())) + .filter(|(_, l)| !l.is_empty() && !l.starts_with('#') && !l.starts_with("//")) + .map(|(n, l)| serde_json::from_str(l).map_err(|e| format!("line {n}: {e}"))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + #[test] + fn tasks_parse_and_resolve_pages() { + let tasks = parse_tasks(TASKS).expect("dataset parses"); + assert!(tasks.len() >= 10); + let mut ids = BTreeSet::new(); + for t in &tasks { + assert!(ids.insert(t.id.clone()), "duplicate id {}", t.id); + assert!(page(&t.page).is_some(), "{}: unknown page {}", t.id, t.page); + assert!(!t.expectations.is_empty(), "{}: no expectations", t.id); + for e in &t.expectations { + assert!(e.weight > 0.0, "{}: non-positive weight", t.id); + } + } + } + + #[test] + fn every_task_covers_completeness_and_noise() { + // Each page has chrome to strip and content to keep; a task that only + // checks one side cannot tell "kept everything" from "kept nothing". + for t in parse_tasks(TASKS).unwrap() { + let metrics: BTreeSet<_> = t.expectations.iter().map(|e| e.metric()).collect(); + assert!(metrics.contains(&Metric::Completeness), "{}", t.id); + assert!(metrics.contains(&Metric::Noise), "{}", t.id); + } + } + + #[test] + fn pages_table_matches_data_dir() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/data/pages"); + let on_disk: BTreeSet = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + let embedded: BTreeSet = PAGES.iter().map(|(n, _)| n.to_string()).collect(); + assert_eq!(on_disk, embedded, "PAGES out of sync with data/pages/"); + let used: BTreeSet = parse_tasks(TASKS) + .unwrap() + .into_iter() + .map(|t| t.page) + .collect(); + assert_eq!(used, embedded, "every page is used by a task"); + } + + #[test] + fn malformed_line_is_an_error() { + let err = parse_tasks("{\"id\":\"x\"}\n").unwrap_err(); + assert!(err.starts_with("line 1:"), "{err}"); + } + + #[test] + fn unknown_check_is_an_error() { + let line = r#"{"id":"x","category":"c","description":"d","page":"p","expectations":[{"check":"eval","text":"x"}]}"#; + assert!(parse_tasks(line).is_err()); + } + + #[test] + fn default_metric_follows_check_and_override_wins() { + let line = r#"{"id":"x","category":"c","description":"d","page":"p","expectations":[ + {"check":"contains","text":"a"}, + {"check":"contains","text":"b","metric":"accuracy","weight":2}]}"# + .replace('\n', ""); + let t = &parse_tasks(&line).unwrap()[0]; + assert_eq!(t.expectations[0].metric(), Metric::Completeness); + assert_eq!(t.expectations[0].weight, 1.0); + assert_eq!(t.expectations[1].metric(), Metric::Accuracy); + assert_eq!(t.expectations[1].weight, 2.0); + } +} diff --git a/crates/markkit-eval/src/lib.rs b/crates/markkit-eval/src/lib.rs new file mode 100644 index 0000000..8d185c3 --- /dev/null +++ b/crates/markkit-eval/src/lib.rs @@ -0,0 +1,14 @@ +//! Mira eval study for markkit extraction quality. +//! +//! Run through the `mira` host CLI (`cargo install mira-cli`): +//! +//! ```text +//! mira list --study-bin markkit-eval +//! mira run --study-bin markkit-eval markkit_extract +//! ``` +//! +//! See `knowledge/operations/eval.md` for the design. + +pub mod checks; +pub mod dataset; +pub mod mira_study; diff --git a/crates/markkit-eval/src/main.rs b/crates/markkit-eval/src/main.rs index e76d643..a7b837b 100644 --- a/crates/markkit-eval/src/main.rs +++ b/crates/markkit-eval/src/main.rs @@ -1,22 +1,23 @@ -use anyhow::Result; -use clap::Parser; +// markkit-eval: mira eval study host. +// +// Advertises markkit's evals to the `mira` CLI over stdio: +// +// mira list --study-bin markkit-eval +// mira run --study-bin markkit-eval markkit_extract +// mira run --study-bin markkit-eval markkit_extract --tag news --format html --out report.html +// +// The `#[eval]` wrappers live in the bin crate so their inventory registrations +// are guaranteed to link into this binary; the work is in +// `markkit_eval::mira_study`. See knowledge/operations/eval.md. -#[derive(Parser)] -#[command(name = "markkit-eval", about = "Evaluate markkit extraction quality")] -struct Cli { - /// Provider (anthropic, openai) - #[arg(long, default_value = "anthropic")] - provider: String, +use markkit_eval::mira_study; +use mira::{Eval, eval}; - /// Evaluation dataset path - #[arg(long)] - dataset: Option, +#[eval] +fn markkit_extract() -> Eval { + mira_study::extract_eval() } -#[tokio::main] -async fn main() -> Result<()> { - let _cli = Cli::parse(); - // TODO: Implement evaluation harness - println!("markkit-eval: evaluation harness not yet implemented"); - Ok(()) +fn main() -> std::io::Result<()> { + mira::Study::registered().serve_blocking() } diff --git a/crates/markkit-eval/src/mira_study.rs b/crates/markkit-eval/src/mira_study.rs new file mode 100644 index 0000000..97fa520 --- /dev/null +++ b/crates/markkit-eval/src/mira_study.rs @@ -0,0 +1,333 @@ +// Mira integration: turns the markkit extraction eval into a mira `Study`. +// +// Three pieces wire markkit into mira (github.com/everruns/mira): +// +// 1. Samples - each JSONL `ExtractTask` becomes a mira `Sample`. The task +// rides in `sample.metadata["task"]` (source of truth for subject and +// scorers); the page HTML in `sample.files["page.html"]`. +// 2. Subject - `extract_subject` runs markkit on the page in-process and puts +// the markdown in `transcript.final_response`, with size/time metrics. +// 3. Scorers - `expectations_scorer` (pass iff every check passes, value = +// weighted pass rate) plus one scorer per quality metric (completeness, +// accuracy, formatting, noise), so reports break quality down by metric. +// +// Decision: a `mode` axis compares extraction configs on the same pages: +// `default` (builder defaults) vs `main_content` (opt-in Readability-style +// main-content detection). One axis, not one eval per config, so reports put +// the configs side by side. +// +// Decision: the thing under test is markkit, not a model, so the eval's only +// target is `Target::cli("markkit")`: always available, no API key, fully +// offline and deterministic. LLM-backed evals (judge, downstream QA) add model +// targets in their own evals. See knowledge/operations/eval.md. + +use std::time::Instant; + +use markkit::Markkit; +use mira::subject::{Subject, subject_fn}; +use mira::{Eval, Sample, Score, Target, Transcript}; + +use crate::checks::{evaluate, per_metric, weighted_rate}; +use crate::dataset::{ExtractTask, Metric, TASKS, page, parse_tasks}; + +/// Workspace path the page is seeded under (visible in mira reports). +pub const PAGE_FILE: &str = "page.html"; + +/// Name of the all-checks scorer. +pub const EXPECTATIONS_SCORER: &str = "markkit_expectations"; + +/// Values of the `mode` axis: which markkit configuration extracts the page. +pub const MODES: [&str; 2] = ["default", "main_content"]; + +/// Build markkit for a `mode` axis value; `None` for an unknown mode. +pub fn markkit_for(mode: &str) -> Option { + match mode { + "default" => Some(Markkit::builder().build()), + "main_content" => Some(Markkit::builder().main_content(true).build()), + _ => None, + } +} + +/// The single target: markkit itself. +pub fn markkit_target() -> Target { + Target::cli("markkit") +} + +/// Build one sample per task. Fails on a malformed dataset or unknown page. +pub fn extract_samples(jsonl: &str) -> Result, String> { + parse_tasks(jsonl)? + .into_iter() + .map(|task| { + let html = page(&task.page) + .ok_or_else(|| format!("{}: unknown page {}", task.id, task.page))?; + let task_value = serde_json::to_value(&task).map_err(|e| e.to_string())?; + Ok(Sample::new(&task.id, &task.description) + .tag(&task.category) + .tag("html") + .meta("category", task.category.clone()) + .meta("page", task.page.clone()) + .meta("task", task_value) + .file(PAGE_FILE, html)) + }) + .collect() +} + +fn task_from_sample(sample: &Sample) -> Result { + let value = sample + .metadata + .get("task") + .cloned() + .ok_or("sample missing task metadata")?; + serde_json::from_value(value).map_err(|e| format!("bad task metadata: {e}")) +} + +/// Run `markkit` on a page and describe the run. +pub fn extract_transcript(markkit: &Markkit, html: &str) -> Transcript { + let start = Instant::now(); + let result = markkit.extract_html_sync(html); + let elapsed = start.elapsed(); + + let markdown = match result { + Ok(md) => md, + // markkit rejecting a page is markkit's failure, not infra: score it. + Err(e) => return Transcript::failed(format!("extraction failed: {e}")), + }; + + let mut t = Transcript::response(markdown.clone()); + t.timing.duration_ms = elapsed.as_millis() as u64; + t.record_metric("extract_us", elapsed.as_micros() as f64); + t.record_metric("input_bytes", html.len() as f64); + t.record_metric("output_bytes", markdown.len() as f64); + if !html.is_empty() { + t.record_metric("size_ratio", markdown.len() as f64 / html.len() as f64); + } + t +} + +/// Subject: extract the sample's page with markkit in the case's `mode` +/// (`default` when the axis is not set). +fn extract_subject() -> impl Subject { + subject_fn(|sample: Sample, cx| async move { + let mode = cx.param("mode").unwrap_or("default").to_string(); + let Some(markkit) = markkit_for(&mode) else { + return Transcript::infra_error(format!("unknown mode {mode:?}")); + }; + match sample.files.get(PAGE_FILE) { + Some(html) => extract_transcript(&markkit, html), + None => Transcript::infra_error("sample missing page.html"), + } + }) +} + +/// Score `transcript` against the sample's expectations, optionally restricted +/// to one metric. `None` metric = every check, pass iff all pass. +fn score(scorer: &str, metric: Option, sample: &Sample, t: &Transcript) -> Score { + if t.errored_infra() { + return Score::na(scorer, "subject infra error"); + } + let task = match task_from_sample(sample) { + Ok(task) => task, + Err(e) => return Score::na(scorer, e), + }; + // A failed extraction produced no markdown: every check runs against "", + // so content checks fail and noise checks pass, like an empty output would. + let results = evaluate(&task.expectations, &t.final_response); + let selected: Vec<_> = results + .iter() + .filter(|r| metric.is_none_or(|m| r.metric == m)) + .collect(); + let Some(value) = weighted_rate(selected.iter().copied()) else { + return Score::na(scorer, "no checks for this metric"); + }; + let failed: Vec<&str> = selected + .iter() + .filter(|r| !r.passed) + .map(|r| r.detail.as_str()) + .collect(); + let passed = selected.len() - failed.len(); + let mut reason = format!("{passed}/{} checks passed", selected.len()); + if metric.is_none() { + let parts: Vec = per_metric(&results) + .into_iter() + .map(|(m, v)| format!("{} {:.0}%", m.as_str(), v * 100.0)) + .collect(); + reason.push_str(&format!(" ({})", parts.join(", "))); + } + if !failed.is_empty() { + reason.push_str(&format!("; failed: {}", failed.join("; "))); + } + Score { + scorer: scorer.to_string(), + value, + pass: failed.is_empty(), + na: false, + reason, + } +} + +/// All checks: pass iff every check passes; value = weighted pass rate. +pub fn expectations_scorer() -> Box { + mira::scorer::scorer(EXPECTATIONS_SCORER, |sample: &Sample, t: &Transcript| { + score(EXPECTATIONS_SCORER, None, sample, t) + }) +} + +/// One metric's checks; N/A when the task has none for it. +pub fn metric_scorer(metric: Metric) -> Box { + let name = metric.as_str(); + mira::scorer::scorer(name, move |sample: &Sample, t: &Transcript| { + score(name, Some(metric), sample, t) + }) +} + +/// The extraction-quality eval: every dataset page through markkit, scored by +/// deterministic checks. Slice with `mira run --tag `. +pub fn extract_eval() -> Eval { + let samples = extract_samples(TASKS).expect("embedded dataset is valid (tested)"); + let mut b = Eval::new("markkit_extract") + .describe("HTML extraction quality: completeness, accuracy, formatting, noise") + .subject(extract_subject()) + .scorer(expectations_scorer()) + .max_turns(1) + .axis("mode", MODES) + .target(markkit_target()); + for m in Metric::ALL { + b = b.scorer(metric_scorer(m)); + } + for s in samples { + b = b.add_sample(s); + } + b.build() +} + +#[cfg(test)] +mod tests { + use super::*; + use mira::RunCx; + + #[test] + fn samples_carry_task_and_page() { + let samples = extract_samples(TASKS).unwrap(); + assert!(samples.len() >= 10); + for s in &samples { + let task = task_from_sample(s).unwrap(); + assert_eq!(s.files[PAGE_FILE], page(&task.page).unwrap()); + assert!(s.tags.contains(&task.category), "{}", s.id); + } + } + + #[test] + fn unknown_page_is_rejected() { + let line = r#"{"id":"x","category":"c","description":"d","page":"nope.html","expectations":[{"check":"contains","text":"a"}]}"#; + assert!(extract_samples(line).unwrap_err().contains("unknown page")); + } + + #[test] + fn eval_builds() { + let _ = extract_eval(); + } + + fn sample(expectations: &str) -> Sample { + let line = format!( + r#"{{"id":"t","category":"c","description":"d","page":"recipe.html","expectations":{expectations}}}"# + ); + extract_samples(&line).unwrap().remove(0) + } + + #[test] + fn scorer_pass_partial_and_metric_breakdown() { + let s = sample( + r#"[{"check":"contains","text":"alpha"},{"check":"contains","text":"zeta"},{"check":"not_contains","text":"ad"}]"#, + ); + let t = Transcript::response("# alpha beta"); + let all = score(EXPECTATIONS_SCORER, None, &s, &t); + assert!(!all.pass); + assert!((all.value - 2.0 / 3.0).abs() < 1e-9); + assert!(all.reason.contains("completeness 50%"), "{}", all.reason); + assert!(all.reason.contains("contains \"zeta\""), "{}", all.reason); + + let noise = score("noise", Some(Metric::Noise), &s, &t); + assert!(noise.pass && noise.value == 1.0); + let fmt = score("formatting", Some(Metric::Formatting), &s, &t); + assert!(fmt.na, "no formatting checks => N/A"); + } + + #[test] + fn infra_error_is_na_but_extraction_failure_is_scored() { + let s = sample(r#"[{"check":"contains","text":"alpha"}]"#); + assert!(score("x", None, &s, &Transcript::infra_error("boom")).na); + let failed = score("x", None, &s, &Transcript::failed("extraction failed")); + assert!(!failed.na && !failed.pass); + } + + #[test] + fn extraction_failure_is_not_infra() { + // Over the input limit: markkit refuses, which must count against it. + let markkit = Markkit::builder().build(); + let huge = "a".repeat(markkit.config().limits.max_input_size + 1); + let t = extract_transcript(&markkit, &huge); + assert!(t.error.is_some() && !t.errored_infra()); + } + + #[tokio::test] + async fn subject_extracts_page_and_records_metrics() { + let s = sample(r#"[{"check":"contains","text":"x"}]"#); + let t = extract_subject() + .run(&s, &RunCx::new(markkit_target())) + .await; + assert!(t.error.is_none()); + assert!(!t.final_response.is_empty()); + for m in ["extract_us", "input_bytes", "output_bytes", "size_ratio"] { + assert!(t.metrics.contains_key(m), "missing metric {m}"); + } + } + + fn cx(mode: &str) -> RunCx { + let mut cx = RunCx::new(markkit_target()); + cx.params.insert("mode".to_string(), mode.to_string()); + cx + } + + #[test] + fn every_mode_builds_and_unknown_is_rejected() { + for m in MODES { + assert!(markkit_for(m).is_some(), "{m}"); + } + assert!(markkit_for("readability").is_none()); + } + + #[tokio::test] + async fn main_content_mode_changes_extraction() { + // The news page wraps the article in chrome; main-content detection + // must drop at least some of it, or the axis measures nothing. + let s = extract_samples(TASKS) + .unwrap() + .into_iter() + .find(|s| s.id == "news_article") + .unwrap(); + let default = extract_subject().run(&s, &cx("default")).await; + let main = extract_subject().run(&s, &cx("main_content")).await; + assert!(default.error.is_none() && main.error.is_none()); + assert_ne!(default.final_response, main.final_response); + } + + #[tokio::test] + async fn unknown_mode_is_infra() { + let s = sample(r#"[{"check":"contains","text":"x"}]"#); + assert!( + extract_subject() + .run(&s, &cx("bogus")) + .await + .errored_infra() + ); + } + + #[tokio::test] + async fn subject_without_page_is_infra() { + let s = Sample::new("bare", "no page"); + let t = extract_subject() + .run(&s, &RunCx::new(markkit_target())) + .await; + assert!(t.errored_infra()); + } +} diff --git a/justfile b/justfile index 77b2295..aa12863 100644 --- a/justfile +++ b/justfile @@ -113,9 +113,18 @@ bench-criterion *args: # ─── Eval ──────────────────────────────────────────────────── -# Run the LLM evaluation harness (stub, not yet implemented) +# Evals run through the mira host CLI (cargo install mira-cli); markkit-eval is +# the study binary. Saved runs: crates/markkit-eval/results (mira.toml). +# See knowledge/operations/eval.md. + +# List advertised evals, samples, scorers, and targets +eval-list: + mira list --study-bin markkit-eval + +# Run the extraction-quality eval. Extra mira flags pass through, e.g. +# just eval --tag docs --format html --out report.html eval *args: - cargo run -p markkit-eval --release -- {{args}} + mira run --study-bin markkit-eval markkit_extract {{args}} # ─── Security & Audit ─────────────────────────────────────── diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index d4c6c09..11701bb 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -54,7 +54,7 @@ are trusted embedder code; the first error aborts and is returned as is | `markkit` | Core library – parsing, conversion, API | | `markkit-cli` | Command-line tool | | `markkit-bench` | Benchmark harness, results committed in `results/` | -| `markkit-eval` | LLM-based quality evaluation | +| `markkit-eval` | Extraction-quality eval study (mira) | | `markkit-python` | Python bindings (PyO3) | | `markkit-js` | Node.js bindings (NAPI-RS) | diff --git a/knowledge/log.md b/knowledge/log.md index b6fd9a9..42c26dc 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -5,6 +5,7 @@ * **MCP server**: `markkit mcp` serves `MarkkitTool` over MCP stdio (initialize, ping, tools/list, tools/call); line size capped from `--max-size`, tool failures as `isError` results. New [MCP Server](integrations/mcp-server.md) concept; MCP messages row in the [Threat Model](security/threat-model.md). * **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. * **Plain text and Markdown input**: text is always on and goes through the IR; Markdown (`markdown` feature) is re-parsed with pulldown-cmark rather than passed through, so raw HTML never reaches output. Neither is sniffed from bytes. Shares the call's `Limits` budget (timeout, nodes, nesting depth). New threat T7 and invariant S18. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). +* **Eval harness on mira**: `markkit-eval` is now a mira study (modeled on bashkit-eval). `markkit_extract` scores markkit's HTML output on ten realistic pages with typed deterministic checks and one scorer per metric; a `mode` axis compares defaults with `main_content` (73.0% vs 86.8% mean, noise 34% vs 82%); dated results committed in `crates/markkit-eval/results/`; gaps recorded (`main_content` drops the forum question, `4.6` over-escaped as `4\.6`, `tfoot`/`caption` dropped, edit links in headings). See [Evaluation Harness](operations/eval.md). ## 2026-10-26 diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index 1af0b2c..77352bb 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -1,45 +1,195 @@ --- type: Test Strategy title: Evaluation Harness -description: LLM-as-judge extraction quality evaluation with its methodology, metrics, and providers. +description: markkit-eval as a mira study, its extraction-quality eval, dataset format, check types, metrics, and baseline. tags: - markkit - eval - - llm + - mira --- # Evaluation Harness ## Status -Not implemented. `crates/markkit-eval` is a CLI stub (`--provider`, `--dataset`) -that prints a placeholder and exits 0. This document is the design it will follow. +`markkit_extract` implemented: an offline, deterministic extraction-quality +eval on the [mira](https://github.com/everruns/mira) eval framework. LLM-backed +evals (LLM-as-judge, downstream QA over a model matrix) are planned as separate +evals in the same study; see [Planned evals](#planned-evals). Modeled on +everruns/bashkit's `bashkit-eval` study. -## Overview +## Architecture -`markkit-eval` measures extraction quality using LLM-as-judge -to compare markkit output against reference extractions. +`markkit-eval` is a **mira study**: a binary that advertises its evals to the +`mira` host CLI over stdio. Mira owns selection, scheduling, saved runs and +reporting (terminal, JSON/JSONL/CSV/JUnit/Markdown/HTML). markkit supplies the +subject under test and the scoring. -## Methodology +``` +mira host CLI ──spawns──▶ markkit-eval (study binary) + │ │ + │ selection (--tag) ├─ Sample (one per JSONL task, page seeded) + │ saved runs / resume ├─ Subject (markkit extract_html_sync) + │ reports └─ Scorers (all checks + one per metric) +``` -1. Curate a dataset of (URL, reference markdown) pairs -2. Extract markdown from each URL using markkit -3. Score extraction quality via LLM comparison -4. Track metrics over time +Code: `crates/markkit-eval/src/` (`dataset.rs`, `checks.rs`, `mira_study.rs`; +`main.rs` holds the `#[eval]` wrappers so their registrations link into the +binary). + +### Key Design Decisions + +1. **The target is markkit, not a model.** `markkit_extract` has one target, + `Target::cli("markkit")`: always available, no API key, offline and + deterministic. Model targets belong to the LLM evals only. +2. **In-process subject.** The subject calls `Markkit::extract_html_sync` with + default settings; no CLI round-trip. Depends only on `mira-eval` (no + `mira-everruns`) and `markkit` with `html` only (no HTTP client). +3. **Typed checks, not `kind:arg` strings.** Page text routinely contains `:`, + which makes string parsing of two-argument checks (`link`, `order`) + ambiguous. Checks are a tagged JSON enum. No regex: page text is data, + never a pattern. +4. **Per-metric scorers.** Every expectation belongs to one metric + (completeness, accuracy, formatting, noise). Mira reports one score per + metric per page plus the all-checks score, so a regression shows which + dimension moved. +5. **Pages are real `.html` files** under `data/pages/` (reviewable, open in a + browser), embedded with `include_str!` through `dataset::PAGES`. A test keeps + the table, the directory and the tasks in sync. +6. **Expectations describe ideal markdown, not current output.** Failing + checks are the point: they are the extraction gaps. Tests do not gate on the + score; they gate on dataset validity and scorer behavior. +7. **Extraction errors are scored, not infra.** markkit refusing a page is + markkit's failure (`Transcript::failed`, checks run against empty output). + Only a malformed sample is an infra error (N/A). + +## Dataset Format (JSONL) + +`crates/markkit-eval/data/extract-tasks.jsonl`, one task per line (`#` and +`//` comment lines skipped; a malformed line is an error, never dropped): + +```json +{"id": "recipe", "category": "recipe", "description": "Recipe with ad slots ...", + "page": "recipe.html", + "expectations": [ + {"check": "heading", "level": 1, "text": "Weeknight Lemon Garlic Pasta", "weight": 2}, + {"check": "line", "text": "- 350 g spaghetti"}, + {"check": "not_contains", "text": "Sponsored"} + ]} +``` + +`page` = file under `data/pages/`; `category` becomes a mira tag (`--tag +recipe`), as does `html`; `weight` defaults to 1; `metric` overrides the +check's default metric. Every task must check both completeness and noise +(test-enforced), so "kept everything" and "kept nothing" both fail. + +## Check Types + +| Check | Fields | Passes when | Default metric | +|-------|--------|-------------|----------------| +| `contains` | `text` | `text` occurs in the output | completeness | +| `not_contains` | `text` | `text` does not occur | noise | +| `line` | `text` | some line equals `text` (trailing whitespace ignored) | formatting | +| `heading` | `level`, `text` | line `#{level} {text}`; level outside 1-6 never passes | formatting | +| `link` | `text`, `url` | `[text](url)` occurs | accuracy | +| `order` | `first`, `then` | both occur, first occurrence of `first` precedes `then` | accuracy | ## Metrics -- **Completeness**: Is all important content preserved? -- **Accuracy**: Is the content faithful to the original? -- **Formatting**: Is the markdown well-structured? -- **Noise**: Is non-content (nav, ads, etc.) excluded? +- **Completeness**: important content preserved. +- **Accuracy**: content faithful (order, link targets). +- **Formatting**: markdown structure right (headings, lists, tables, fences). +- **Noise**: non-content (nav, ads, banners, comments, widgets) excluded. + +Scorers: `markkit_expectations` (pass iff every check passes; value = weighted +pass rate; reason lists per-metric rates and failed checks) and one scorer per +metric (N/A when a task has no check for it). Subject metrics +(`transcript.metrics`): `extract_us`, `input_bytes`, `output_bytes`, +`size_ratio`. + +## Dataset + +Ten pages, one per category, each with realistic chrome around the content: +`news` (cookie banner, inline ad, newsletter form, related aside), `docs` +(sidebar, option table, TOML/Rust code, edit link, pager), `blog` (share +buttons, comments, popup), `commerce` (breadcrumbs, buy box, row-header spec +table, recommendations, promo), `recipe` (ad slots, lists, rating widget), +`reference` (infobox, citations, TOC, edit links, categories), `forum` (votes, +accepted answer with code, sidebar), `changelog` (nested lists, strikethrough, +issue links, CTA), `data` (captioned table with thead/tbody/tfoot), `legacy` +(table layout, font tags, unclosed `

`, hit counter). + +## Mode Axis + +`mode` axis (`mira run --axis mode=main_content`): `default` = builder +defaults; `main_content` = `main_content(true)`, the opt-in Readability-style +detector. Same pages and checks, so reports compare the two side by side. +Unknown mode = infra error. + +## Results + +Committed under `crates/markkit-eval/results/`: `markkit_extract-.md` +(summary: per mode, per metric, per page, failed checks) and the matching mira +JSON report (per-case scores, transcripts). Mira run folders stay local +(gitignored). Re-run `just eval` after extraction changes and commit a new +dated pair when scores move. + +## Baseline (2026-10-27) + +| mode | pages passing | overall | completeness | accuracy | formatting | noise | +|---|---|---|---|---|---|---| +| `default` | 0/10 | 73.0% | 92.3% | 100% | 95.6% | 34.3% | +| `main_content` | 2/10 | 86.8% | 88.3% | 90.0% | 88.1% | 82.1% | + +First run (before IR v2, default only): 72.5%; row-header tables then +collapsed to one column and dropped every ``, fixed by IR v2 row-header +`` mapping. Open gaps: + +- **Noise (default)**: `