diff --git a/CHANGELOG.md b/CHANGELOG.md index bac0c8c..cc9e4e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,16 @@ different event from one that moved because it was wrong. ranking (`site/search.js`) runs in the browser with no library. `scripts/check_search.mjs` holds the ranking to its cases and every index entry to a page and id that exist. +- The explainer opens with the example report's result drawn as a card (the + measured ECE against its floor's 95th percentile, and the verdict), written + at build time from the report's own line so it reads the same with scripts + off, and keeps its contents in a rail beside the text on wide screens. +- The calculator and planner each run in their own worker and can be + cancelled; their inputs are kept in the address, so a link reproduces a + result, and each result has "Copy link" and "Copy result" buttons. +- Every page carries a Content-Security-Policy that allows nothing but the + site's own files (no inline script or style), and `check_site_links.mjs` + fails a page that lacks it or carries anything it would block. ### Changed @@ -59,6 +69,17 @@ different event from one that moved because it was wrong. ### Fixed +- Every validation message on the site's calculator and planner read + "[object Object]" (#25). Messages now say what is wrong, mark the field + invalid, and are tied to it for screen readers; a result is announced as one + line rather than the whole result block (#53). +- A failure to draw the worked example was reported as the example being + missing; the page now says which happened. `floor.js` turned a bin count of 0 + into 10 where the Python refuses it, and now refuses it too. The parity + check holds the worked example's floor to 1e-9, not only to the four + decimals the report prints (#55). +- A link to a later section of the explainer (`#planner`) stopped short of it, + because the worked example grows the page after it loads. - The inconclusive verdict read as a pass. "Not distinguishable from a perfectly calibrated model at this sample size" is what the arithmetic establishes and close to the opposite of what it means, and readers took it diff --git a/scripts/build_site.py b/scripts/build_site.py index 2c3903a..5e5414b 100644 --- a/scripts/build_site.py +++ b/scripts/build_site.py @@ -56,7 +56,7 @@ from typing import Any, TypedDict from plumbline.cli import app -from plumbline.metrics.calibration import synthetic_floor +from plumbline.metrics.calibration import calibration_floor, synthetic_floor from plumbline.runner.execute import RunResult ROOT = Path(__file__).resolve().parent.parent @@ -292,6 +292,7 @@ def build_example() -> dict[str, Any]: n_bins = int(ece_match["bins"]) accuracy = sum(outcomes) / len(outcomes) summary = synthetic_floor(len(probabilities), n_bins=n_bins, accuracy=accuracy)["ece"] + real = calibration_floor(run.probabilities(), n_bins=n_bins)["ece"] return { "about": ( @@ -317,6 +318,10 @@ def build_example() -> dict[str, Any]: "python": { "accuracy": accuracy, "summary_floor": {"mean": summary.mean, "p95": summary.p95}, + # The floor from the rows themselves, unrounded, so the page's + # calibrationFloor is held to 1e-9 and not only to the 4 decimals + # the report prints. + "calibration_floor": {"mean": real.mean, "p95": real.p95}, }, } @@ -572,10 +577,22 @@ def social_meta(title: str, description: str, url: str) -> str: ) +#: The Content Security Policy every page carries. Everything the site loads is +#: its own: no inline script or style, no other origin, and data: only for the +#: inline favicon. ``site/index.html`` carries the same tag, which the build +#: requires, and ``scripts/check_site_links.mjs`` checks every page for it. +CSP = ( + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; " + "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; " + "object-src 'none'; base-uri 'none'; form-action 'none'" +) +CSP_META = f'' + PAGE = """ +{csp} {title} | plumbline @@ -622,6 +639,7 @@ def _page( ) -> str: """One doc-shaped page: header, docs sidebar, the body, and its contents list.""" return PAGE.format( + csp=CSP_META, title=html.escape(title), description=html.escape(description, quote=True), meta=meta, @@ -702,11 +720,71 @@ def write_404(out: Path) -> None: HEADER_SLOT = "" -def explainer_page(source: str) -> str: - """The explainer as written, with the shared header in its slot.""" - if source.count(HEADER_SLOT) != 1: - raise BuildError(f"site/index.html must contain {HEADER_SLOT} exactly once") - return source.replace(HEADER_SLOT, site_header("", "calculator")) +#: Where the example's result card goes, in the explainer's opening block. +CARD_SLOT = "" + + +def example_card(report: dict[str, Any]) -> str: + """The example report's headline result, drawn at build time. + + ``report`` is the ``report`` block of ``example-run.json``: the figures parsed + from the report's own ECE line, so the card says exactly what the report + says, and says it with scripts off. The gauge puts the measured ECE against + the calibrated-model floor's 95th percentile on one scale. + """ + ece, p95, n = float(report["ece"]), float(report["floor_p95"]), int(report["n"]) + inconclusive = "INCONCLUSIVE" in report["ece_line"] + width, scale = 320.0, max(ece, p95) * 1.3 + x_ece, x_p95 = width * ece / scale, width * p95 / scale + anchor = "start" if x_ece < 60 else "end" if x_ece > width - 60 else "middle" + gauge = ( + f'' + f'' + f'' + f'' + f'' + f"measured {ece:.4f}" + f'' + f"floor, 95th percentile {p95:.4f}" + ) + if inconclusive: + chip = 'INCONCLUSIVE' + says = ( + f"The measured figure sits inside what a perfectly calibrated model scores on " + f"{n} rows, so this dataset establishes nothing either way." + ) + else: + chip = 'Distinguishable' + says = ( + f"The measured figure is above what a perfectly calibrated model scores on {n} " + "rows nineteen times in twenty." + ) + return ( + '
\n' + f'
The example report, {n} rows
\n' + '
\n' + f'

Measured ECE {ece:.4f}

\n' + '

Calibrated-model floor, 95th percentile ' + f'{p95:.4f}

\n' + "
\n" + f" {gauge}\n" + f'

{chip} {says} ' + 'See it derived from the rows.

\n' + "
" + ) + + +def explainer_page(source: str, report: dict[str, Any]) -> str: + """The explainer as written, with the shared header and the example card in their slots.""" + for slot in (HEADER_SLOT, CARD_SLOT): + if source.count(slot) != 1: + raise BuildError(f"site/index.html must contain {slot} exactly once") + if source.count(CSP_META) != 1: + raise BuildError("site/index.html must carry the site's Content-Security-Policy tag") + page = source.replace(HEADER_SLOT, site_header("", "calculator")) + return page.replace(CARD_SLOT, example_card(report)) #: The explainer's name in search results. @@ -789,7 +867,7 @@ def main() -> int: prov = provenance() rendered = render_docs(prov) source = (SITE / "index.html").read_text(encoding="utf-8") - explainer = explainer_page(source) + explainer = explainer_page(source, example["report"]) index = search_index(rendered, source) except BuildError as problem: print(f"site build refused: {problem}", file=sys.stderr) diff --git a/scripts/check_floor_parity.mjs b/scripts/check_floor_parity.mjs index 3929909..d94d945 100644 --- a/scripts/check_floor_parity.mjs +++ b/scripts/check_floor_parity.mjs @@ -72,6 +72,10 @@ if (examplePath) { for (const key of ["mean", "p95"]) { const gap = Math.abs(summary[key] - example.python.summary_floor[key]); if (!(gap <= tolerance)) failures.push(`worked example: summary floor ${key} off by ${gap}`); + // The line above agrees to 4 decimals; this holds the floor from the rows + // to the same 1e-9 as every other case. + const real = Math.abs(band[key] - example.python.calibration_floor[key]); + if (!(real <= tolerance)) failures.push(`worked example: calibration floor ${key} off by ${real}`); } console.log(`worked example: ${derived.slice(0, 110)}...`); } diff --git a/scripts/check_site_links.mjs b/scripts/check_site_links.mjs index b2d7c46..e7aaa19 100644 --- a/scripts/check_site_links.mjs +++ b/scripts/check_site_links.mjs @@ -13,6 +13,9 @@ // - a page loads anything (script, stylesheet, image, frame) from another origin, // - a page lacks its canonical, Open Graph, or Twitter card tags, or its // canonical and og:url do not name the page itself, +// - a page does not carry exactly the site's Content-Security-Policy tag, or +// has an inline script, an inline style block, or a style attribute (which +// that policy would block, so the page would silently break), // - og.png is not a 1200x630 PNG, // - the 404 page is missing, links anywhere that does not resolve, or (live) // a missing path is not answered with it and a 404 status. @@ -23,6 +26,13 @@ import path from "node:path"; const SITE_URL = "https://tmhsdigital.github.io/plumbline/"; const SITE_PATH = new URL(SITE_URL).pathname; // "/plumbline/" +// Stated here rather than read from build_site.py, so a change to the policy +// is a change to what this check requires. +const CSP = + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; " + + "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; " + + "object-src 'none'; base-uri 'none'; form-action 'none'"; + const target = process.argv[2]; if (!target) { console.error("usage: check_site_links.mjs "); @@ -119,6 +129,15 @@ function checkSocial(sitePath, html) { } } +function checkPolicy(where, html) { + const tags = [...html.matchAll(//g)]; + if (tags.length !== 1) problems.push(`${where}: ${tags.length} Content-Security-Policy tags, expected 1`); + else if (tags[0][1] !== CSP) problems.push(`${where}: Content-Security-Policy is not the site's policy`); + if (/]*\bsrc=)[^>]*>/i.test(html)) problems.push(`${where}: an inline - + -
-
+
+

A calibration claim has a floor

A perfectly calibrated model does not score an Expected Calibration Error of zero. On a finite sample it scores something above zero, and on a few hundred rows that something is often larger than the differences people report. Below a certain row count, a calibration claim is not measurable at all. This page lets you check where your own numbers sit, without installing anything.

- + +

+ Check your numbers + Read the methodology +

+ +

The argument

ECE sorts predictions into bins by the probability the model gave, and in each bin compares @@ -144,12 +98,13 @@

Floor calculator

+
-