From a7142dbb90459a06409a79ecd64a74ffa3aa8c44 Mon Sep 17 00:00:00 2001 From: Manuel Klarmann Date: Mon, 16 Nov 2026 18:05:22 +0100 Subject: [PATCH 01/14] A frameless page for a site's own page, and the matrix of every item MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page is now one body for both uses. `page.body(data)` is a single `
` with the file inside it as JSON: no html, head or body, and no id. A site holds it and loads diagnostics.js and diagnostics.css from `page.assets()`. `page.write` wraps the same body in a page of its own with `standalone.css`. - The style is scoped: every rule is under `.sentier-diagnostics`. It sets no font and nothing on `:root` or `body`, and resets what a site does to bare tables. Colours are `--sd-*` that take the site's tokens (`--sd-ink: var(--ink, ...)`). Up, down and noise keep their own colours. - The script mounts into every `[data-sentier-diagnostics]`. It reads the data from the body, a global or `?file=`, and sets the document title only on a page of its own. - The matrix of every item (`sentier-diagnostics-matrix/1`, new module `matrix`): a local file of items may name one, and `chain.build_with_matrix` writes it. `matrix.against` works every code out again and holds the file's counts, attribution and `items_count` to it exactly. The page loads it after drawing, as `matrix.js` beside the page or as JSON over http, and refuses one of another file. - The grid shows after ÷ before, coloured by direction and shaded by band. It sorts by a heading (furthest first), filters by name, group, class and verdict, and draws 300 rows at a time. - The item opens in a drawer. "Why it differs" comes first: the item's own explanations, then those of a step or indicator it moved in. - CLI: `check` holds a matrix beside its file to it; `page` writes it; `example` also writes the local example. The local example is committed in examples/local/. Taken from the trial dashboard of sentier-bafu-agribalyse: its look and structure only. None of its values, and not its two definitions of agreement, hand-written labels or a "none" over a top-N list. Co-Authored-By: Claude Opus 5.5 --- GLOSSARY.md | 4 + README.md | 17 +- SPEC.md | 126 +- examples/local/diagnostics.json | 1112 +++++++++++++ examples/local/matrix.json | 50 + src/sentier_diagnostics/chain.py | 45 +- src/sentier_diagnostics/cli.py | 42 +- src/sentier_diagnostics/example.py | 12 +- src/sentier_diagnostics/fileformat.py | 21 +- src/sentier_diagnostics/matrix.py | 413 +++++ src/sentier_diagnostics/page.py | 174 ++- src/sentier_diagnostics/page/body.html | 106 ++ src/sentier_diagnostics/page/diagnostics.css | 249 +-- src/sentier_diagnostics/page/diagnostics.js | 1478 +++++++++++------- src/sentier_diagnostics/page/index.html | 112 +- src/sentier_diagnostics/page/standalone.css | 37 + tests/js/dom.js | 111 +- tests/js/page.test.js | 174 ++- tests/test_matrix_page.py | 315 ++++ tests/test_modules.py | 6 +- tests/test_repository.py | 12 +- 21 files changed, 3754 insertions(+), 862 deletions(-) create mode 100644 examples/local/diagnostics.json create mode 100644 examples/local/matrix.json create mode 100644 src/sentier_diagnostics/matrix.py create mode 100644 src/sentier_diagnostics/page/body.html create mode 100644 src/sentier_diagnostics/page/standalone.css create mode 100644 tests/test_matrix_page.py diff --git a/GLOSSARY.md b/GLOSSARY.md index 00ac68f..dc39666 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -25,3 +25,7 @@ The words this repository uses, each in one line. [SPEC.md](SPEC.md) says the re - **Breakdown**: what one item's result is made of, before and after one step. - **Tier**: whether a file lists items (`items`) or holds counts only (`counts`). - **Audience**: whether a file's writer says it may be published (`public`) or not (`local`). +- **Listed items**: the items a file holds, the worst first, up to `listed`. +- **Matrix**: every item of a local file, listed or not, in a second file the page loads after it has drawn. +- **Body**: the page as one element a site holds in its own page; its style stays inside it. +- **Drawer**: the panel of one item, at the right of the page, opened from a row. diff --git a/README.md b/README.md index 3f940e8..e08f4da 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,10 @@ sentier-agribalyse and others): - a chain of steps from the published results to the current build, each held against what it claimed to move, with the whole difference split between the steps (exactly, up to the rounding the page shows) and what nothing explains named; -- a static page that shows it, with no build step, no server and no network. +- a page that shows it: a frameless body a site holds in its own page, with a style + scoped to it that takes the site's tokens, or a page of its own with no build step, no + server and no network; a local file can carry a matrix of every item, which the page + loads after it has drawn. The published results are checked once, at the start (step 0). Every later step is compared with the step before it. @@ -26,7 +29,17 @@ database. Scoring and the readers of each database stay in the projects that use PYTHONPATH=src python3 -m sentier_diagnostics example --out build/example ``` -Then open `build/example/page/index.html`. The example is made up. +Then open `build/example/page/index.html`, or `build/example/local/page/index.html` for a +local file whose page loads the matrix of every item. The example is made up. + +In a site's own page: + +```python +from sentier_diagnostics import page + +html = page.body(data, title=False) # one
, the file inside +assets = page.assets() # diagnostics.js and diagnostics.css, to copy +``` Install it in a project, pinned to a tag: diff --git a/SPEC.md b/SPEC.md index fbe6969..a03b894 100644 --- a/SPEC.md +++ b/SPEC.md @@ -130,6 +130,7 @@ control characters, bounded lengths. | `explanations` | Section 8 | | `notes` | Up to 20 texts | | `made_by` | `"sentier-diagnostics 0.1.0"` | +| `matrix` | Optional. `{"file": name, "items": n}`: the full matrix of every item, a second file beside this one (section 4.5); null or absent for none. Only a local file of items names one, and `items` is `items_count.all` | A **block** counts one indicator in one step, or over the whole way: @@ -189,6 +190,40 @@ many items as a file may (5,000); a state's identity has up to 12 entries. Ids a 100 characters, labels 80, texts 600 and an explanation's text 2,000. The limits are in `fileformat.py`, and a refusal names the limit it met. +### 4.5 The matrix of every item + +A file lists the items worth a look, up to `listed`. A project looking at its own build +wants every item: `chain.build_with_matrix` writes them in a second file of format +`sentier-diagnostics-matrix/1`, which the page loads after it has drawn. Only a local file +of items has one; a public file keeps its listed items only. + +```json +{"format": "sentier-diagnostics-matrix/1", + "of": {"title": "...", "states": ["a", "b", "c"], "indicators": ["x:climate", "x:water"], "steps": ["s0", "s1"]}, + "items": [["p01", "Wheat", "FR", "crops", "kg"], ...], + "values": [[0.52, 0.53, 0.47, 1.2, 1.2, null], ...], + "classes": ["=Dd===", ...], + "verdicts": ["a.==", ...]} +``` + +One row per item, in the order of the listed items (worst first): `[id, name, place, +group, unit]`; the results per indicator and state, indicator after indicator; per +indicator one code per step and one for the whole way; and, where a step has a claim, per +indicator one verdict code per step (`.` for a step without a claim; `verdicts` is null when +no step has one). The class codes are `=` agrees, `~` within noise, `0` near zero, `-` +missing, and for a pair that moved its class in upper case up and lower case down: `D` +differs, `B` beyond 10x, `R` zero reference, `T` zero result, `S` sign change. The verdict +codes are `a` as claimed, `o` off claim, `n` not claimed, `c` claimed, not moved, `~` within +noise, `=` unchanged, `?` not judged. A matrix holds no band: it follows from the results. + +`matrix.errors(m)` checks the shape and never raises. `matrix.against(m, data)` works every +code out again with the file's floors, tolerances and noise (and every verdict where the +claim's lists are in the file), and holds the file to it: with every item at hand, its +counts, attribution, `items_count.all` and `qualifying` must be what the matrix holds, +exactly, and every listed item must be in the matrix with the same name and results. +`sentier-diagnostics check` does both when the matrix lies beside its file. A matrix holds +up to 200,000 items and 64 MB. + ## 5. The counting rule ### 5.1 One class per pair @@ -405,17 +440,40 @@ sentier-agribalyse's sums over ecoinvent processes and may not be published. ## 10. Views -A static page: `index.html`, `diagnostics.js`, `diagnostics.css`. No build step, no -server, no network, no font or script from elsewhere. `sentier-diagnostics page FILE --out -FOLDER` writes it with the file beside it as `data.js`, so it opens from disk; served over -HTTP it can also read `?file=...`, and it opens a file a reader chooses (read in the -browser, sent nowhere). **The page works out no class, no direction and no count of its -own.** It reads them from the file, which the validator has checked, and only adds the -file's counts up: the classes that moved into "moved", and a step's counts over its -indicators. A file read by `?file=` or -chosen by a reader has not passed through the validator: the page checks its shape only, -says so above everything it draws, and refuses a file it cannot draw instead of drawing -part of it. Whatever the file says, only an `https://` address becomes a link. +**One body for a site and for a page of its own.** `page.body(data)` is the diagnostics as +one element, `
`, with the file +(and its breakdowns) inside it as JSON: no ``, `` or ``, and no id, so a +site may hold it anywhere, even twice. The site loads `diagnostics.js` and +`diagnostics.css` from `page.assets()`; the script mounts into every such element (a site +that draws it later marks it `="manual"` and calls `SentierDiagnostics.mount(element, +{data})`). `data-sd-base` names where the files it loads lie, and `title=False` leaves the +heading to the site. The script sets the document's title only on a page of its own. + +**The style is scoped.** Every rule of `diagnostics.css` is under `.sentier-diagnostics`; +it sets no font, nothing on `:root`, `html` or `body`, and resets inside its root what a +site does to bare tables (borders, phone layouts that turn rows into blocks). Its colours +are `--sd-*` properties that take the site's tokens where the site sets them +(`--sd-ink: var(--ink, #1d232a)`, and so on for `--muted`, `--surface`, `--paper-deep`, +`--line`, `--line-strong`, `--index`, `--fail`, `--radius`, `--t-*`, `--s*`); up, down and +noise keep colours of their own, so that a direction never takes a site's accent. A test +reads the sheet and holds this. + +`sentier-diagnostics page FILE --out FOLDER` writes a page of its own: `index.html` with +the body in it, the two assets, and `standalone.css`, which sets the tokens and font a site +would. It opens from disk; no build step, no server, no network, no font or script from +elsewhere. Where the file names a matrix that lies beside it, the page writes it as +`matrix.js`, which loads from disk too; a site serving `matrix.json` needs nothing else, as +the page fetches it over HTTP. Served over HTTP the page can also read `?file=...`, and it +opens a file a reader chooses (read in the browser, sent nowhere). **The page works out no +class, no direction and no count of its own.** It reads them from the file, which the +validator has checked, and only adds the file's counts up: the classes that moved into +"moved", and a step's counts over its indicators. A file read by `?file=` or chosen by a +reader has not passed through the validator: the page checks its shape only, says so above +everything it draws, and refuses a file it cannot draw instead of drawing part of it. A +matrix is held to the file it says it is of (title, states, indicators, steps, number of +items, the length of every row) before a row of it is drawn; its codes are checked by +`sentier-diagnostics check`, not again in the browser. Whatever the file says, only an +`https://` address becomes a link. The text a page renders comes from the file or from words chosen to be neutral: it avoids stage, triage, ledger, layer, upstream, provider and candidate (a test holds this). @@ -441,16 +499,31 @@ sizes lie apart (`|log10(after/before)|`, or the relative difference where there ratio). The page always says how many it shows of how many, and how many items worth a look the file does not hold. -### 10.5 Grid -Items by indicator for one comparison, filtered by text, class, verdict and group, paged by -100 with "show more". The reader's **size filter** ("size at least ×1.1 ... ×10") hides -smaller changes here and in the worst list; zeros, sign changes and missing values always -show, and the page says the filter changes no count. +### 10.5 Matrix +Items by indicator for one comparison: each cell is the result after divided by the result +before, where the two share a sign (`1 + 3e−7` rather than a `1` that is not one), and what +the class says where they do not (from 0, to 0, + → −, missing, ≈ 0, =). The cell's colour +is its direction's and its strength the band's; for a matrix row, which holds no band, the +band is named from the results as the rule names it (a test holds the two equal). The ratio +and the shading are a display: they change no class. The listed items, or every item once +the matrix is loaded, a dot marking the listed ones; the page says which it holds and why +(none named, loading, loaded, or the reason it was not loaded). A heading sorts by how far +the two lie apart in that indicator, the furthest first (the order of 10.4), or by name; +filters by text, group, class and verdict; 300 rows at a time with "show more". The +reader's **size filter** ("size at least ×1.1 ... ×10") hides smaller changes here and in +the worst list; zeros, sign changes and missing values always show, and the page says the +filter changes no count. ### 10.6 Item -Both sides in every state, the class, verdict and signed share of each step, the share -nothing explains and the sum of the shares (or why there are none: section 7), its -explanations, and its breakdowns where they are in the file and fit the item and step. +A drawer at the right (the whole window on a phone), opened from a row of the matrix or the +worst list, closed with Close or Escape. First **why it differs**: the explanations about +the item, then those about a step or an indicator in which it moved (said to be so, as they +may not be about it), or "none given" when the file holds neither; the file holds every +explanation, so this "none" is never said over a part of a list. Then per indicator, the +one whose whole way lies furthest apart first: both sides in every state, the class, +verdict and signed share of each step, the share nothing explains and the sum of the shares +(or why there are none: section 7). Then its breakdowns where they are in the file and fit +the item and step. ### 10.7 Explanations and rule Every explanation with its origin badge; the rule with its tolerances, noise, floors and @@ -508,7 +581,14 @@ line on ordinary mistakes, never a traceback; the summary escapes the file's tex Markdown; noise is taken beyond the floor, and a scope entry read from tables without `expect` may move; the page draws the example whole under Node with a small stand-in for the DOM (`tests/js/dom.js`), loads nothing from elsewhere and writes no markup from data; -the committed example is what the code makes, holds every class and every verdict, and no +the page takes the matrix the Python writes and refuses one of another file, loads it as a +script or over HTTP after it has drawn, draws 300 rows at a time, sorts by a heading, and +opens the drawer with "why it differs" first (node tests); `matrix.against` finds a changed +code, direction, result, verdict or count (`tests/test_matrix_page.py`); `page.body` is one +root with no frame and no id, and its data cannot end the element that holds it; every +selector of the style is inside the root, with no font and nothing on `:root` or `body`; +the committed examples (the public file, and the local file with its matrix in +`examples/local/`) are what the code makes, the public one holds every class and every verdict, and no tracked file carries an identifier-shaped string (`tests/test_repository.py`). ## 12. Out of scope @@ -568,7 +648,8 @@ sentier-bafu-agribalyse wrote their requirements. What became of each: | Closer and further not at the head of a later step | bafu, ADR 0009 | yes: counted, never shown as better | | A chain of more than two states, each step's share | all three | yes | | `listed` named and counted apart from what moved | bafu | yes: `items_count` | -| The page fetches its data as a file | bafu | yes: `data.js` beside the page, or `?file=` | +| The page fetches its data as a file | bafu | yes: the data inside the body, or `window.SENTIER_DIAGNOSTICS` from a site's own `data.js` (`page.as_script`), or `?file=` | +| Every item, not only the listed ones, on a local page | bafu | yes: the matrix (section 4.5), loaded after the page is drawn | | Noise per item | bafu | no: later | | The number of digits written | bafu | values are written unrounded; a rounded source is a tolerance basis | | Symmetric bands, floor of 1 % of the median per indicator | bafu | yes, and per unit | @@ -576,7 +657,8 @@ sentier-bafu-agribalyse wrote their requirements. What became of each: | A test that the script agrees with the Python rule | bafu | yes: shared cases | | A formatter target at the lowest Python | bafu | yes: `py310`, CI runs 3.10 to 3.14 | | Standard library only; assets as package data with their paths | bafu, bafu-agribalyse | yes: `page.assets()` | -| A page without a frame, with a list of tokens for a site's frame | bafu, bafu-agribalyse | no: later (the page has its own tokens) | +| A page without a frame, with a list of tokens for a site's frame | bafu, bafu-agribalyse | yes: `page.body`, a scoped style whose `--sd-*` take the site's tokens (section 10) | +| The look of the trial dashboard: a sortable grid, a drawer for an item | bafu-agribalyse | yes: its look and structure; none of its values, and not its two definitions of agreement, its hand-written labels or a "none" over a top-N list (section 10) | | Reading version 1 and 2 files, or a converter | bafu | no: see section 13 and the migration plan | | A tier a site check can refuse on | bafu, bafu-agribalyse | yes: `tier` (items or counts) and `audience` (public or local) | | Join accounting per reference | bafu-agribalyse | yes: `join` per state | diff --git a/examples/local/diagnostics.json b/examples/local/diagnostics.json new file mode 100644 index 0000000..def56d0 --- /dev/null +++ b/examples/local/diagnostics.json @@ -0,0 +1,1112 @@ +{ + "format": "sentier-diagnostics/1", + "tier": "items", + "audience": "local", + "title": "A made-up chain: published results to the current build", + "about": "Made-up results of ten items in three indicators. Nobody's data.", + "item": { + "one": "product", + "many": "products" + }, + "indicators": [ + { + "id": "x:climate", + "name": "Climate change", + "short": "Climate", + "unit": "kg CO2 eq" + }, + { + "id": "x:water", + "name": "Water use", + "short": "Water", + "unit": "m3" + }, + { + "id": "x:tox", + "name": "Human toxicity", + "short": "Toxicity", + "unit": "CTUh" + } + ], + "rule": { + "floor_factor": 0.01, + "floors_from": "published", + "beyond": 10.0, + "total_tolerance": 0.0005, + "floors": [ + { + "indicator": "x:climate", + "unit": "MJ", + "value": 0.0003, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "h", + "value": 0.42, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:climate", + "unit": "kg", + "value": 0.0049, + "basis": "median", + "n": 8 + }, + { + "indicator": "x:water", + "unit": "MJ", + "value": 4.000000000000001e-06, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "h", + "value": 0.009000000000000001, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:water", + "unit": "kg", + "value": 0.011000000000000001, + "basis": "median", + "n": 7 + }, + { + "indicator": "x:tox", + "unit": "MJ", + "value": 3e-11, + "basis": "median", + "n": 1 + }, + { + "indicator": "x:tox", + "unit": "h", + "value": 0.0, + "basis": "none", + "n": 0 + }, + { + "indicator": "x:tox", + "unit": "kg", + "value": 1.95e-09, + "basis": "median", + "n": 8 + } + ] + }, + "states": [ + { + "id": "published", + "label": "Published", + "about": "What the publisher released.", + "kind": "published", + "identity": { + "release": "1.0" + }, + "join": { + "compared": 10, + "unmatched": 1, + "ambiguous": 0, + "note": "One published row names no item of the build." + } + }, + { + "id": "repro", + "label": "Recomputed", + "about": null, + "kind": "build", + "identity": { + "commit": "0000001" + }, + "join": null + }, + { + "id": "fix-1", + "label": "With fix 1", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002" + }, + "join": null + }, + { + "id": "rebuilt", + "label": "Rebuilt", + "about": null, + "kind": "build", + "identity": { + "commit": "0000002", + "machine": "second" + }, + "join": null + }, + { + "id": "mapping-2", + "label": "With mapping 2", + "about": null, + "kind": "build", + "identity": { + "commit": "0000003" + }, + "join": null + } + ], + "steps": [ + { + "id": "step-0", + "from": "published", + "to": "repro", + "label": "Reproduction", + "about": "The released data computed again with the same method.", + "kind": "reproduction", + "explained": true, + "link": null, + "claim": { + "text": "Nothing moves beyond the rounding of the published results.", + "by": { + "kind": "code", + "name": "reproduction rule", + "model": null, + "run_by": null + }, + "reach": null, + "scope": [] + }, + "tolerance": { + "value": 0.0005, + "basis": "published rounding", + "note": "four significant digits" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "fix-1", + "from": "repro", + "to": "fix-1", + "label": "Fertiliser amounts of two cereals", + "about": "The nitrogen amounts of wheat and rice were per hectare, not per kilogram.", + "kind": "fix", + "explained": true, + "link": "https://example.org/findings/fix-1", + "claim": { + "text": "Wheat and rice go down in climate change, by 30 % at most. Nothing else moves.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "reach": 2, + "scope": [ + { + "indicators": [ + "x:climate" + ], + "items": [ + "p01", + "p05" + ], + "direction": "down", + "largest": 0.3, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "rebuilt", + "from": "fix-1", + "to": "rebuilt", + "label": "Built again on another machine", + "about": "The same state, built on a second machine.", + "kind": "rebuild", + "explained": false, + "link": null, + "claim": null, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": false, + "between": null, + "by": null, + "note": null + } + }, + { + "id": "mapping-2", + "from": "rebuilt", + "to": "mapping-2", + "label": "Mapping change 2", + "about": "Five items take another background dataset.", + "kind": "mapping", + "explained": true, + "link": null, + "claim": { + "text": "Items whose supply chain reaches the changed links may move in water use; maize must.", + "by": { + "kind": "code", + "name": "reach of the changed links", + "model": null, + "run_by": null + }, + "reach": 4, + "scope": [ + { + "indicators": [ + "x:water" + ], + "items": [ + "p02", + "p04", + "p07", + "p10" + ], + "direction": "any", + "largest": null, + "expect": "may" + }, + { + "indicators": [ + "x:water" + ], + "items": [ + "p02" + ], + "direction": "up", + "largest": 0.5, + "expect": "must" + } + ] + }, + "tolerance": { + "value": 1e-05, + "basis": "default", + "note": "one order of magnitude above the solver noise of two builds" + }, + "noise": { + "measured": true, + "between": [ + "fix-1", + "rebuilt" + ], + "by": { + "x:climate": 0.0, + "x:water": 0.02439024390243894, + "x:tox": 0.0 + }, + "note": "One pair of builds of one state." + } + } + ], + "counts": { + "steps": { + "step-0": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 9, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 9, + "not judged": 1 + }, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 2 + }, + "closer": null, + "further": null + } + }, + "fix-1": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 2 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 1, + "off claim": 1, + "not claimed": 1, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 0 + }, + "closer": 0, + "further": 3 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 10, + "not judged": 0 + }, + "closer": 0, + "further": 0 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 8, + "not judged": 2 + }, + "closer": 0, + "further": 0 + } + }, + "rebuilt": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 10, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 8, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 2, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 0, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": 0, + "further": 0 + } + }, + "mapping-2": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 0, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 1, + "differs": 0, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": { + "as claimed": 1, + "off claim": 0, + "not claimed": 0, + "claimed, not moved": 1, + "within noise": 1, + "unchanged": 6, + "not judged": 1 + }, + "closer": 0, + "further": 1 + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 7, + "within noise": 0, + "differs": 1, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 1, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": { + "as claimed": 0, + "off claim": 0, + "not claimed": 2, + "claimed, not moved": 0, + "within noise": 0, + "unchanged": 7, + "not judged": 1 + }, + "closer": 0, + "further": 2 + } + } + }, + "total": { + "x:climate": { + "pairs": 10, + "classes": { + "agrees": 4, + "within noise": 0, + "differs": 3, + "beyond 10x": 0, + "zero reference": 0, + "zero result": 1, + "sign change": 1, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 0, + "after": 1, + "both": 0 + }, + "directions": { + "up": 2, + "down": 3 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 1, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 0, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:water": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 1, + "zero reference": 0, + "zero result": 0, + "sign change": 0, + "near zero": 0, + "missing": 1 + }, + "missing": { + "before": 1, + "after": 0, + "both": 0 + }, + "directions": { + "up": 3, + "down": 0 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 0, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 1, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 1 + }, + "verdicts": null, + "closer": null, + "further": null + }, + "x:tox": { + "pairs": 10, + "classes": { + "agrees": 6, + "within noise": 0, + "differs": 2, + "beyond 10x": 0, + "zero reference": 1, + "zero result": 0, + "sign change": 0, + "near zero": 1, + "missing": 0 + }, + "missing": { + "before": 0, + "after": 0, + "both": 0 + }, + "directions": { + "up": 2, + "down": 1 + }, + "bands": { + "down, factor over 10": 0, + "down, factor 2-10": 0, + "down, factor 1.25-2": 1, + "down, factor 1.1-1.25": 0, + "down, factor 1-1.1": 0, + "up, factor 1-1.1": 1, + "up, factor 1.1-1.25": 0, + "up, factor 1.25-2": 0, + "up, factor 2-10": 0, + "up, factor over 10": 0 + }, + "verdicts": null, + "closer": null, + "further": null + } + } + }, + "attribution": { + "x:climate": { + "dominant": { + "step-0": 0, + "fix-1": 3, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + }, + "x:water": { + "dominant": { + "step-0": 0, + "fix-1": 0, + "rebuilt": 1, + "mapping-2": 1 + }, + "not_attributable": 1, + "spread": 0, + "unexplained_moved": 1 + }, + "x:tox": { + "dominant": { + "step-0": 1, + "fix-1": 0, + "rebuilt": 0, + "mapping-2": 2 + }, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0 + } + }, + "items": [ + {"id": "p08", "name": "Compost, used as fertiliser", "place": "CH", "group": "waste", "unit": "kg", "values": {"x:climate": [-0.5, -0.5, -0.5, -0.5, 0.2], "x:water": [0.05, 0.05, 0.05, 0.05, 0.05], "x:tox": [-2e-06, -2e-06, -2e-06, -2e-06, -4e-06]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "sign change", "sign change"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "differs", "differs"]}, "directions": {"x:climate": [null, null, null, "up", "up"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down", "down"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "down, factor 1.25-2", "down, factor 1.25-2"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p09", "name": "Machine hour, harvesting", "place": "CH", "group": "services", "unit": "h", "values": {"x:climate": [42.0, 42.00042, 42.00042, 42.00042, 42.00042], "x:water": [0.9, 0.9, 0.9, 0.9, 0.9], "x:tox": [0.0, 0.0, 0.0, 0.0, 4e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "zero reference", "zero reference"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, "up", "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not claimed"]}, "breakdowns": null}, + {"id": "p07", "name": "Heat from wood chips", "place": "CH", "group": "energy", "unit": "MJ", "values": {"x:climate": [0.03, 0.03, 0.03, 0.03, 0.0], "x:water": [0.0004, 0.0004, 0.0004, 0.0004, 0.0004], "x:tox": [3e-09, 3e-09, 3e-09, 3e-09, 3e-09]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "zero result", "zero result"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, null, null, "down", "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "not claimed"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p10", "name": "Tomato, greenhouse", "place": "NL", "group": "crops", "unit": "kg", "values": {"x:climate": [1.3, 1.3, 1.3, 1.3, 1.3], "x:water": [0.02, 0.02, 0.02, 0.02, 0.5], "x:tox": [2.1e-07, 2.3e-07, 2.3e-07, 2.3e-07, 2.3e-07]}, "classes": {"x:climate": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:water": ["agrees", "agrees", "agrees", "beyond 10x", "beyond 10x"], "x:tox": ["differs", "agrees", "agrees", "agrees", "differs"]}, "directions": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up", "up"], "x:tox": ["up", null, null, null, "up"]}, "bands": {"x:climate": [null, null, null, null, null], "x:water": [null, null, null, "up, factor over 10", "up, factor over 10"], "x:tox": ["up, factor 1-1.1", null, null, null, "up, factor 1-1.1"]}, "verdicts": {"x:climate": ["unchanged", "unchanged", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "as claimed"], "x:tox": ["not claimed", "unchanged", null, "unchanged"]}, "breakdowns": null}, + {"id": "p05", "name": "Rice", "place": "IT", "group": "crops", "unit": "kg", "values": {"x:climate": [1.9, 1.9, 0.95, 0.95, 0.95], "x:water": [3.4, 3.4, 3.4, 3.4, 3.4], "x:tox": [1e-12, 1e-12, 1e-12, 1e-12, 3e-12]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "agrees", "agrees", "agrees"], "x:tox": ["near zero", "near zero", "near zero", "near zero", "near zero"]}, "directions": {"x:climate": [null, "down", null, null, "down"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "down, factor 1.25-2", null, null, "down, factor 1.25-2"], "x:water": [null, null, null, null, null], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "off claim", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "unchanged"], "x:tox": ["not judged", "not judged", null, "not judged"]}, "breakdowns": null}, + {"id": "p03", "name": "Barley", "place": "DE", "group": "crops", "unit": "kg", "values": {"x:climate": [0.48, 0.47999, 0.575988, 0.575988, 0.575988], "x:water": [1.1, 1.1, 1.1, null, 1.3], "x:tox": [1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07, 1.9e-07]}, "classes": {"x:climate": ["agrees", "differs", "agrees", "agrees", "differs"], "x:water": ["agrees", "agrees", "missing", "missing", "differs"], "x:tox": ["agrees", "agrees", "agrees", "agrees", "agrees"]}, "directions": {"x:climate": [null, "up", null, null, "up"], "x:water": [null, null, null, null, "up"], "x:tox": [null, null, null, null, null]}, "bands": {"x:climate": [null, "up, factor 1.1-1.25", null, null, "up, factor 1.1-1.25"], "x:water": [null, null, null, null, "up, factor 1.1-1.25"], "x:tox": [null, null, null, null, null]}, "verdicts": {"x:climate": ["unchanged", "not claimed", null, "unchanged"], "x:water": ["unchanged", "unchanged", null, "not judged"], "x:tox": ["unchanged", "unchanged", null, "unchanged"]}, "breakdowns": null} + ], + "items_count": { + "all": 10, + "qualifying": 10, + "listed": 6, + "not_listed": 4 + }, + "explanations": [ + { + "about": { + "step": "fix-1", + "indicator": "x:climate", + "item": null + }, + "finding": "reason", + "text": "The fertiliser amounts were per hectare; divided by the yield they are per kilogram, which lowers the nitrogen field emissions of both cereals.", + "by": { + "kind": "person", + "name": "A. Author", + "model": null, + "run_by": null + }, + "source": { + "link": null, + "doc": "docs/findings/fix-1.md" + }, + "not_checked": null + }, + { + "about": { + "step": null, + "indicator": "x:tox", + "item": "p05" + }, + "finding": "reason", + "text": "The toxicity results of rice lie far below one hundredth of the median result of the items measured per kilogram: they are counted as near zero.", + "by": { + "kind": "code", + "name": "floor rule", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + }, + { + "about": { + "step": "mapping-2", + "indicator": "x:water", + "item": "p10" + }, + "finding": "reason", + "text": "The new mapping links tomatoes to a greenhouse heated with fresh water from a well, which multiplies the water use.", + "by": { + "kind": "model", + "name": null, + "model": "example-model-1", + "run_by": "example maintainers (agent-run)" + }, + "source": null, + "not_checked": "Nobody compared the greenhouse's water balance with a second source." + }, + { + "about": { + "step": "step-0", + "indicator": "x:tox", + "item": "p08" + }, + "finding": "none found", + "text": "Compost's toxicity result fell to almost nothing in the reproduction. The recipe, the background datasets and the method were compared; none of them explains it.", + "by": { + "kind": "person", + "name": "B. Reviewer", + "model": null, + "run_by": null + }, + "source": null, + "not_checked": null + } + ], + "notes": [ + "A made-up example: nobody's data." + ], + "made_by": "sentier-diagnostics 0.1.0", + "matrix": { + "file": "matrix.json", + "items": 10 + } +} diff --git a/examples/local/matrix.json b/examples/local/matrix.json new file mode 100644 index 0000000..7945f66 --- /dev/null +++ b/examples/local/matrix.json @@ -0,0 +1,50 @@ +{"format": "sentier-diagnostics-matrix/1", "of": {"title": "A made-up chain: published results to the current build", "states": ["published", "repro", "fix-1", "rebuilt", "mapping-2"], "indicators": ["x:climate", "x:water", "x:tox"], "steps": ["step-0", "fix-1", "rebuilt", "mapping-2"]}, + "items": [ + ["p08","Compost, used as fertiliser","CH","waste","kg"], + ["p09","Machine hour, harvesting","CH","services","h"], + ["p07","Heat from wood chips","CH","energy","MJ"], + ["p10","Tomato, greenhouse","NL","crops","kg"], + ["p05","Rice","IT","crops","kg"], + ["p03","Barley","DE","crops","kg"], + ["p01","Wheat","FR","crops","kg"], + ["p04","Oats","DE","crops","kg"], + ["p06","Rye","PL","crops","kg"], + ["p02","Maize","FR","crops","kg"] + ], + "values": [ + [-0.5,-0.5,-0.5,-0.5,0.2,0.05,0.05,0.05,0.05,0.05,-2e-06,-2e-06,-2e-06,-2e-06,-4e-06], + [42.0,42.00042,42.00042,42.00042,42.00042,0.9,0.9,0.9,0.9,0.9,0.0,0.0,0.0,0.0,4e-07], + [0.03,0.03,0.03,0.03,0.0,0.0004,0.0004,0.0004,0.0004,0.0004,3e-09,3e-09,3e-09,3e-09,3e-09], + [1.3,1.3,1.3,1.3,1.3,0.02,0.02,0.02,0.02,0.5,2.1e-07,2.3e-07,2.3e-07,2.3e-07,2.3e-07], + [1.9,1.9,0.95,0.95,0.95,3.4,3.4,3.4,3.4,3.4,1e-12,1e-12,1e-12,1e-12,3e-12], + [0.48,0.47999,0.575988,0.575988,0.575988,1.1,1.1,1.1,null,1.3,1.9e-07,1.9e-07,1.9e-07,1.9e-07,1.9e-07], + [0.52,0.520013,0.468012,0.468012,0.468012,1.2,1.2,1.2,1.2000001,1.2000001,2e-07,2.0000002e-07,2.0000002e-07,2.0000002e-07,2.0000002e-07], + [0.45,0.45,0.45,0.45,0.45,1.0,1.0,1.0,1.025,1.0455,1.7e-07,1.7e-07,1.7e-07,1.7e-07,1.7e-07], + [0.43,0.43,0.43,0.43,null,null,1.15,1.15,1.15,1.15,2.1e-07,2.1e-07,2.1e-07,2.1e-07,2.1e-07], + [0.41,0.41,0.41,0.41,0.41,2.5,2.500025,2.500025,2.500025,2.500025,1.8e-07,1.8e-07,1.8e-07,1.8e-07,1.8e-07] + ], + "classes": [ + "===SS========dd", + "==========000RR", + "===tt==========", + "========BBD===D", + "=d==d=====00000", + "=D==D==--D=====", + "=d==d==========", + "=======D~D=====", + "===---===-=====", + "===============" + ], + "verdicts": [ + "==.n==.===.n", + "==.===.=??.n", + "==.n==.===.=", + "==.===.an=.=", + "=o.===.=??.?", + "=n.===.?==.=", + "=a.===.===.=", + "==.===.~==.=", + "==.??=.===.=", + "==.===.c==.=" + ] +} diff --git a/src/sentier_diagnostics/chain.py b/src/sentier_diagnostics/chain.py index b207c86..8ee9252 100644 --- a/src/sentier_diagnostics/chain.py +++ b/src/sentier_diagnostics/chain.py @@ -26,7 +26,7 @@ from collections.abc import Iterable, Mapping from pathlib import Path -from . import __version__, claims, fileformat, rule +from . import __version__, claims, fileformat, matrix, rule FILE = "diagnostics.json" @@ -170,7 +170,21 @@ def _values(values, states, indicators) -> dict: return found -def build( +def build(title: str, **given) -> dict: + """A whole chain, counted, judged, attributed and checked: the file, without a matrix. + The keywords are those of `_build`, whose docstring says what each one holds.""" + return _build(title, **given)[0] + + +def build_with_matrix(title: str, *, matrix_file: str = "matrix.json", **given) -> tuple[dict, dict]: + """The file and its full matrix (`sentier_diagnostics.matrix`): every item with its + results and codes, the worst first, in a second file named `matrix_file` that the file + names. Only a local file of items has one.""" + data, found = _build(title, matrix_file=matrix_file, **given) + return data, found + + +def _build( title: str, *, indicators: Iterable, @@ -190,8 +204,10 @@ def build( explanations: Iterable[dict] = (), notes: Iterable[str] = (), breakdowns: Mapping[str, Mapping[str, str]] | None = None, -) -> dict: - """A whole chain, counted, judged, attributed and checked. + matrix_file: str | None = None, +) -> tuple[dict, dict | None]: + """A whole chain, counted, judged, attributed and checked, and its matrix where + `matrix_file` names one. - `indicators`: `(id, name, short, unit)` or dicts with those keys. - `states`: `state(...)`, in order; `steps`: `step(...)`, one fewer than the states. @@ -209,7 +225,10 @@ def build( - `audience`: "local" (the default: for the people who made it) or "public" (its writer says it may be published). The tool checks only that it is one of the two. - `breakdowns`: `{item id: {step id or "total": file name}}`. + - `matrix_file`: the name of the full matrix, for a local file of items; None for none. """ + if matrix_file is not None and (tier != "items" or audience != "local"): + raise ValueError("only a local file of items has a matrix: a public file keeps the listed items only") meta = [_indicator(i) for i in indicators] ids = [i["id"] for i in meta] if len(steps) != len(states) - 1: @@ -371,8 +390,8 @@ def worst(item): shown = qualifying[: max(0, listed)] if tier == "items" else [] breakdowns = breakdowns or {} any_claim = any(s.get("claim") for s in steps) - out_items = [] - for item in shown: + + def record_of(item): about_item = names.get(item) or {} record = { "id": item, @@ -394,7 +413,9 @@ def worst(item): record["bands"][indicator] = [r[4] for r in row] if any_claim: record["verdicts"][indicator] = [r[5] for r in row[:-1]] - out_items.append(record) + return record + + out_items = [record_of(item) for item in shown] data = { "format": fileformat.FORMAT, @@ -431,10 +452,18 @@ def worst(item): "notes": list(notes), "made_by": f"sentier-diagnostics {__version__}", } + if matrix_file is not None: + data["matrix"] = {"file": matrix_file, "items": len(items)} errors = fileformat.errors(data) if errors: raise ValueError("not a diagnostics file: " + "; ".join(errors[:10])) - return data + if matrix_file is None: + return data, None + full = matrix.make(data, [record_of(item) for item in sorted(items, key=worst)]) + errors = matrix.errors(full) or matrix.against(full, data) + if errors: + raise ValueError("not a matrix of this file: " + "; ".join(errors[:10])) + return data, full #: Stands in for the list of items while the rest of a file is written. A file's texts hold diff --git a/src/sentier_diagnostics/cli.py b/src/sentier_diagnostics/cli.py index 465b62e..69bbac8 100644 --- a/src/sentier_diagnostics/cli.py +++ b/src/sentier_diagnostics/cli.py @@ -1,10 +1,11 @@ """The command `sentier-diagnostics`: check, build, summarise and show diagnostics files. - sentier-diagnostics check FILE [FILE ...] exit 1 if any file is not one a page shows - sentier-diagnostics build DESCRIPTION --out FILE - sentier-diagnostics summary FILE - sentier-diagnostics page FILE --out FOLDER - sentier-diagnostics example --out FOLDER +sentier-diagnostics check FILE [FILE ...] exit 1 if any file is not one a page shows + (a matrix beside a file is checked against it) +sentier-diagnostics build DESCRIPTION --out FILE +sentier-diagnostics summary FILE +sentier-diagnostics page FILE --out FOLDER +sentier-diagnostics example --out FOLDER """ from __future__ import annotations @@ -13,7 +14,7 @@ import sys from pathlib import Path -from . import __version__, breakdown, chain, example, fileformat, from_tables, page, summary +from . import __version__, breakdown, chain, example, fileformat, from_tables, matrix, page, summary def check(paths) -> int: @@ -27,11 +28,15 @@ def check(paths) -> int: continue if isinstance(data, dict) and data.get("format") == breakdown.FORMAT: errors = breakdown.errors(data) + elif isinstance(data, dict) and str(data.get("format", "")).startswith(matrix.FAMILY): + errors = matrix.errors(data) # against its file when the file is checked else: errors = fileformat.errors(data) size = Path(path).stat().st_size if size > fileformat.MAX_BYTES: - errors.append(f"{size} bytes, and a page holds {fileformat.MAX_BYTES} at most: list fewer items") + errors.append( + f"{size} bytes, and a page holds {fileformat.MAX_BYTES} at most: list fewer items" + ) if not errors: try: found, absent = page.named_breakdowns(data, Path(path).parent) @@ -41,6 +46,14 @@ def check(paths) -> int: errors += page.breakdown_errors(data, found) for name in absent: print(f"note {path}: names the breakdown {name}, which is not beside it (not checked)") + if data.get("matrix"): + try: + if page.named_matrix(data, Path(path).parent) is None: + print( + f"note {path}: names the matrix {data['matrix']['file']}, which is not beside it (not checked)" + ) + except ValueError as error: + errors.append(str(error)) if errors: failed += 1 print(f"FAIL {path}") @@ -82,9 +95,14 @@ def main(argv=None) -> int: if args.command == "page": data = chain.read(args.file) found, absent = page.named_breakdowns(data, args.file.parent) - where = page.write(data, args.out, found) + full = page.named_matrix(data, args.file.parent) + where = page.write(data, args.out, found, full) for name in absent: print(f"note: the breakdown {name} is not beside the file; the page says so") + if data.get("matrix") and full is None: + print( + f"note: the matrix {data['matrix']['file']} is not beside the file; the page shows the listed items and says so" + ) print(f"wrote {where}") return 0 if args.command == "example": @@ -93,7 +111,13 @@ def main(argv=None) -> int: one = example.wheat_breakdown() breakdown.write(one, args.out / "p01-fix-1.json") where = page.write(data, args.out / "page", {"p01-fix-1.json": one}) - print(f"wrote {args.out / chain.FILE} and {where}") + local, full = example.build_local() + chain.write(local, args.out / "local" / chain.FILE) + matrix.write(full, args.out / "local" / local["matrix"]["file"]) + local_page = page.write(local, args.out / "local" / "page", None, full) + print( + f"wrote {args.out / chain.FILE} and {where}; the local example and its matrix: {local_page}" + ) return 0 except (ValueError, OSError) as error: print(f"error: {error}", file=sys.stderr) diff --git a/src/sentier_diagnostics/example.py b/src/sentier_diagnostics/example.py index cde40e3..e6719bf 100644 --- a/src/sentier_diagnostics/example.py +++ b/src/sentier_diagnostics/example.py @@ -202,7 +202,17 @@ def explanations() -> list[dict]: def build(**more) -> dict: """The made-up chain, counted and checked.""" - return chain.build( + return _made(chain.build, **more) + + +def build_local() -> tuple[dict, dict]: + """The same chain as a local file that lists the six worst products, and its full matrix + of all ten: what a project keeps on its own machine.""" + return _made(chain.build_with_matrix, audience="local", listed=6) + + +def _made(builder, **more): + return builder( "A made-up chain: published results to the current build", about="Made-up results of ten items in three indicators. Nobody's data.", indicators=INDICATORS, diff --git a/src/sentier_diagnostics/fileformat.py b/src/sentier_diagnostics/fileformat.py index 43a886a..648070e 100644 --- a/src/sentier_diagnostics/fileformat.py +++ b/src/sentier_diagnostics/fileformat.py @@ -39,6 +39,8 @@ FLOOR_BASES = ("median", "declared", "none") ORIGIN_KINDS = ("person", "code", "model") TOTAL = "total" +#: Keys a file may leave out; each is null where it says nothing. +OPTIONAL_KEYS = {"matrix"} #: The largest file a page reads: it holds the whole file. MAX_BYTES = 12 * 1024 * 1024 @@ -795,7 +797,7 @@ def _errors(data, found: list[str]) -> list[str]: if isinstance(said, str) and said.startswith(FAMILY): return [f"format {said!r}: this reader knows {FORMAT} only"] return [f"not a diagnostics file of format {FORMAT}"] - found += [f"a key this format does not know: {key}" for key in sorted(set(data) - KEYS)] + found += [f"a key this format does not know: {key}" for key in sorted(set(data) - KEYS - OPTIONAL_KEYS)] found += [f"a key this format requires: {key}" for key in sorted(KEYS - set(data))] if found: return found @@ -831,4 +833,21 @@ def _errors(data, found: list[str]) -> list[str]: _attribution(data, found, indicators, steps, blocks) items = _items(data, found, indicators, states, steps, floors, tier, blocks) _explanations(data, found, indicators, steps, set(items), tier) + _matrix(data, found) return found + + +def _matrix(data, found) -> None: + """A matrix holds every item with its results: only a local file of items may name one.""" + named = data.get("matrix") + if named is None: + return + if not (isinstance(named, dict) and set(named) == {"file", "items"} and isinstance(named["file"], str) + and FILE_NAME.fullmatch(named["file"]) and is_count(named["items"])): # fmt: skip + found.append("`matrix` is not {file, items} with a file name, or null") + return + if data["tier"] != "items" or data["audience"] != "local": + found.append("only a local file of items names a matrix: a public file keeps the listed items only") + count = data["items_count"] + if isinstance(count, dict) and named["items"] != count.get("all"): + found.append(f"`matrix` says {named['items']} items, the file counts {count.get('all')}") diff --git a/src/sentier_diagnostics/matrix.py b/src/sentier_diagnostics/matrix.py new file mode 100644 index 0000000..f7c3e0d --- /dev/null +++ b/src/sentier_diagnostics/matrix.py @@ -0,0 +1,413 @@ +"""The full matrix of a local file, `sentier-diagnostics-matrix/1`: every item, not only the +listed ones. + +A diagnostics file lists the items worth a look, the worst first, up to `listed` (200 by +default), so that it stays small enough for a page to hold. A project looking at its own +build wants every item: the matrix is that, in a second file the page loads after it has +drawn. It is compact (one row per item, the indicators and states named once) and it holds +only what the rule gives: the results, and one code per pair for its class and direction, +and per step with a claim one code for its verdict. + +Only a local file may name a matrix (`"matrix": {"file": ..., "items": n}`); a public file +keeps the listed items only. `errors(m)` checks the shape; `against(m, data)` works every +code out again from the results and the diagnostics file's own rule, and holds the file's +counts, attribution and `items_count` to what the matrix holds: with every item at hand, +they must match exactly. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +from . import claims, fileformat, rule + +FORMAT = "sentier-diagnostics-matrix/1" +FAMILY = "sentier-diagnostics-matrix/" +KEYS = {"format", "of", "items", "values", "classes", "verdicts"} +OF_KEYS = {"title", "states", "indicators", "steps"} + +#: A matrix is loaded after the page is drawn, not held inside it: it may be larger. +MAX_BYTES = 64 * 1024 * 1024 +MAX_ITEMS = 200_000 + +#: One character per pair: its class, and for a pair that moved its direction (upper case +#: up, lower case down). +CODES = { + (rule.AGREES, None): "=", + (rule.WITHIN_NOISE, None): "~", + (rule.NEAR_ZERO, None): "0", + (rule.MISSING, None): "-", + (rule.DIFFERS, rule.UP): "D", + (rule.DIFFERS, rule.DOWN): "d", + (rule.BEYOND, rule.UP): "B", + (rule.BEYOND, rule.DOWN): "b", + (rule.ZERO_REFERENCE, rule.UP): "R", + (rule.ZERO_REFERENCE, rule.DOWN): "r", + (rule.ZERO_RESULT, rule.UP): "T", + (rule.ZERO_RESULT, rule.DOWN): "t", + (rule.SIGN_CHANGE, rule.UP): "S", + (rule.SIGN_CHANGE, rule.DOWN): "s", +} +DECODE = {code: pair for pair, code in CODES.items()} + +#: One character per pair and step with a claim: its verdict. "." for a step without one. +VERDICT_CODES = { + claims.AS_CLAIMED: "a", + claims.OFF_CLAIM: "o", + claims.NOT_CLAIMED: "n", + claims.NOT_MOVED: "c", + claims.NOISE: "~", + claims.UNCHANGED: "=", + claims.NOT_JUDGED: "?", +} +NO_CLAIM = "." +VERDICT_DECODE = {code: verdict for verdict, code in VERDICT_CODES.items()} + + +def code_of(kind: str, way: str | None) -> str: + """The code of a class and direction. + + >>> code_of("differs", "up"), code_of("agrees", None), code_of("sign change", "down") + ('D', '=', 's') + """ + return CODES[(kind, way)] + + +def make(data: Mapping, rows: list[dict]) -> dict: + """A matrix of a diagnostics file from its rows, as `chain.build_with_matrix` gives them: + `{id, name, place, group, unit, values: {indicator: [per state]}, classes, directions, + verdicts}` with classes, directions and verdicts per indicator as the file lists them.""" + ids = [i["id"] for i in data["indicators"]] + steps = data["steps"] + out = { + "format": FORMAT, + "of": { + "title": data["title"], + "states": [s["id"] for s in data["states"]], + "indicators": ids, + "steps": [s["id"] for s in steps], + }, + "items": [], + "values": [], + "classes": [], + "verdicts": [] if any(s["claim"] for s in steps) else None, + } + for row in rows: + out["items"].append([row["id"], row["name"], row["place"], row["group"], row["unit"]]) + out["values"].append([v for i in ids for v in row["values"][i]]) + out["classes"].append( + "".join( + code_of(kind, way) for i in ids for kind, way in zip(row["classes"][i], row["directions"][i]) + ) + ) + if out["verdicts"] is not None: + out["verdicts"].append( + "".join( + NO_CLAIM if said is None else VERDICT_CODES[said] + for i in ids + for said in row["verdicts"][i] + ) + ) + return out + + +def dumps(m: Mapping) -> str: + """The text of a matrix: one line per item, so that a diff names the items.""" + head = {k: v for k, v in m.items() if k not in ("items", "values", "classes", "verdicts")} + lines = [json.dumps(head, ensure_ascii=False, allow_nan=False)[:-1] + ","] + for key in ("items", "values", "classes", "verdicts"): + rows = m[key] + if rows is None: + lines.append(f' "{key}": null' + ("," if key != "verdicts" else "")) + continue + body = ",\n".join( + " " + json.dumps(r, ensure_ascii=False, allow_nan=False, separators=(",", ":")) for r in rows + ) + lines.append(f' "{key}": [\n{body}\n ]' + ("," if key != "verdicts" else "")) + return "\n".join(lines) + "\n}\n" + + +def write(m: Mapping, path) -> Path: + """Write a matrix, its shape checked first; refuse one larger than a page loads.""" + found = errors(m) + if found: + raise ValueError("not a matrix: " + "; ".join(found[:10])) + text = dumps(m) + size = len(text.encode("utf-8")) + if size > MAX_BYTES: + raise ValueError(f"the matrix would be {size} bytes, and a page loads {MAX_BYTES} at most") + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def read(path, data: Mapping) -> dict: + """A matrix, read and checked against its checked diagnostics file; raises ValueError + with every error found.""" + path = Path(path) + if path.stat().st_size > MAX_BYTES: + raise ValueError(f"{path}: larger than {MAX_BYTES} bytes") + m = fileformat.load_json(path) + found = errors(m) or against(m, data) + if found: + raise ValueError(f"{path}: " + "; ".join(found[:10])) + return m + + +def _is_row(row) -> bool: + return ( + isinstance(row, list) + and len(row) == 5 + and fileformat.is_text(row[0], fileformat.MAX_ID) + and fileformat.is_text(row[1], 300) + and fileformat.is_optional(row[2], 60) + and fileformat.is_optional(row[3], fileformat.MAX_LABEL) + and fileformat.is_optional(row[4], fileformat.MAX_WORD) + ) + + +def errors(m) -> list[str]: + """Everything wrong with the shape of a matrix; nothing for one a page can load. It never + raises: a value it cannot read is a fault like any other.""" + try: + return _errors(m) + except Exception: # noqa: BLE001 - a backstop: the checks above it name the faults they know + return [fileformat.STOPPED] + + +def _errors(m) -> list[str]: + if not isinstance(m, Mapping): + return ["not a matrix: not an object"] + said = m.get("format") + if said != FORMAT: + if isinstance(said, str) and said.startswith(FAMILY): + return [f"format {said!r}: this reader knows {FORMAT} only"] + return [f"not a matrix of format {FORMAT}"] + found = [f"a key this format does not know: {k}" for k in sorted(set(m) - KEYS)] + found += [f"a key this format requires: {k}" for k in sorted(KEYS - set(m))] + if found: + return found + of = m["of"] + if not ( + isinstance(of, dict) + and set(of) == OF_KEYS + and fileformat.is_text(of["title"], 120) + and all(isinstance(of[k], list) and all(fileformat.is_text(x, fileformat.MAX_ID) for x in of[k]) + for k in ("states", "indicators", "steps")) + ): # fmt: skip + return ["`of` is not {title, states, indicators, steps}"] + nstates, nind, nsteps = len(of["states"]), len(of["indicators"]), len(of["steps"]) + if nsteps != nstates - 1 or not nind: + return ["`of`: one step fewer than states, and at least one indicator"] + items = m["items"] + if not isinstance(items, list) or len(items) > MAX_ITEMS: + return [f"`items` is not a list of at most {MAX_ITEMS} items"] + n = len(items) + for key in ("values", "classes"): + if not isinstance(m[key], list) or len(m[key]) != n: + found.append(f"`{key}` is not one row per item") + has_claims = m["verdicts"] is not None + if has_claims and (not isinstance(m["verdicts"], list) or len(m["verdicts"]) != n): + found.append("`verdicts` is not one row per item, or null") + if found: + return found + seen = set() + for at, row in enumerate(items): + if not _is_row(row): + return [f"item {at}: not [id, name, place, group, unit]"] + if row[0] in seen: + found.append(f"item {row[0]!r} is in the matrix twice") + seen.add(row[0]) + values = m["values"][at] + if not (isinstance(values, list) and len(values) == nind * nstates + and all(v is None or rule.is_value(v) for v in values)): # fmt: skip + return [f"item {row[0]!r}: not one finite number or null per indicator and state"] + codes = m["classes"][at] + if not ( + isinstance(codes, str) and len(codes) == nind * (nsteps + 1) and all(c in DECODE for c in codes) + ): + return [f"item {row[0]!r}: not one class code per indicator, step and the whole way"] + if has_claims: + said = m["verdicts"][at] + if not (isinstance(said, str) and len(said) == nind * nsteps + and all(c in VERDICT_DECODE or c == NO_CLAIM for c in said)): # fmt: skip + return [f"item {row[0]!r}: not one verdict code per indicator and step"] + return found + + +def against(m: Mapping, data: Mapping) -> list[str]: + """Everything a checked matrix and its checked diagnostics file disagree on. Every code + is worked out again with the file's floors, tolerances and noise, and, where a claim's + lists are in the file, every verdict; the file's counts, attribution and `items_count` + must be what the matrix holds, exactly.""" + try: + return _against(m, data) + except Exception: # noqa: BLE001 - a backstop, as in `errors` + return [fileformat.STOPPED] + + +def _against(m: Mapping, data: Mapping) -> list[str]: + found = [] + named = data.get("matrix") + if not named: + return ["the diagnostics file names no matrix"] + ids = [i["id"] for i in data["indicators"]] + order = [s["id"] for s in data["states"]] + steps = data["steps"] + of = {"title": data["title"], "states": order, "indicators": ids, "steps": [s["id"] for s in steps]} + if m["of"] != of: + return ["it is the matrix of another file: its title, states, indicators or steps differ"] + n = len(m["items"]) + if named["items"] != n: + found.append(f"the diagnostics file says {named['items']} items, the matrix holds {n}") + if data["items_count"]["all"] != n: + found.append(f"the diagnostics file counts {data['items_count']['all']} items, the matrix holds {n}") + nstates, nsteps = len(order), len(steps) + floors = {(f["indicator"], f["unit"]): f["value"] for f in data["rule"]["floors"]} + published_first = data["states"][0]["kind"] == "published" + listed = {item["id"]: item for item in data["items"]} + + # what the counts and attribution must be, counted from every item + def empty(k): + return { + "pairs": 0, + "classes": dict.fromkeys(rule.CLASSES, 0), + "missing": dict.fromkeys(fileformat.MISSING_SIDES, 0), + "directions": dict.fromkeys(rule.DIRECTIONS, 0), + "bands": dict.fromkeys(rule.BANDS, 0), + "verdicts": dict.fromkeys(claims.VERDICTS, 0) if k is not None and steps[k]["claim"] else None, + "closer": 0 if published_first and k not in (None, 0) else None, + "further": 0 if published_first and k not in (None, 0) else None, + } + + counted = {(k, i): empty(k) for k in [*range(nsteps), None] for i in ids} + attributed = { + i: { + "dominant": {s["id"]: 0 for s in steps}, + "not_attributable": 0, + "spread": 0, + "unexplained_moved": 0, + } + for i in ids + } + qualifying = 0 + for at, (item_id, name, place, group, unit) in enumerate(m["items"]): + where = f"item {item_id!r}" + flat, codes = m["values"][at], m["classes"][at] + said_verdicts = m["verdicts"][at] if m["verdicts"] is not None else None + worth = False + values_of = {} + for x, indicator in enumerate(ids): + values = flat[x * nstates : (x + 1) * nstates] + values_of[indicator] = values + floor = floors.get((indicator, unit)) + if floor is None: + return [f"{where}, {indicator}: the file holds no floor for unit {unit!r}"] + kinds = [] + for k in [*range(nsteps), None]: + at_k = nsteps if k is None else k + before, after = (values[0], values[-1]) if k is None else (values[k], values[k + 1]) + if k is None: + tol, level = data["rule"]["total_tolerance"], None + else: + tol = steps[k]["tolerance"]["value"] + noise = steps[k]["noise"] + level = (noise["by"] or {}).get(indicator) if noise["measured"] else None + kind, way, band = rule.judge(before, after, floor=floor, tolerance=tol, noise=level) + code = codes[x * (nsteps + 1) + at_k] + if code != code_of(kind, way): + step_name = "the whole way" if k is None else steps[k]["id"] + return [ + f"{where}, {indicator}, {step_name}: written as {DECODE[code]}, the rule gives {kind}/{way}" + ] + kinds.append(kind) + block = counted[(k, indicator)] + block["pairs"] += 1 + block["classes"][kind] += 1 + if kind == rule.MISSING: + side = ( + "both" + if before is None and after is None + else "before" + if before is None + else "after" + ) + block["missing"][side] += 1 + if way: + block["directions"][way] += 1 + if band: + block["bands"][band] += 1 + if block["closer"] is not None and kind in rule.MOVES and values[0] is not None: + was, now = abs(before - values[0]), abs(after - values[0]) + block["closer"] += now < was + block["further"] += now > was + worth = worth or kind not in rule.QUIET + if k is None: + continue + step_claim = steps[k]["claim"] + vcode = said_verdicts[x * nsteps + k] if said_verdicts is not None else NO_CLAIM + if step_claim is None: + if vcode != NO_CLAIM: + return [f"{where}, {indicator}, {steps[k]['id']}: a verdict without a claim"] + continue + said = VERDICT_DECODE.get(vcode) + if said is None or said not in fileformat.ALLOWED_VERDICTS[kind]: + return [f"{where}, {indicator}, {steps[k]['id']}: {kind!r} cannot be {said!r}"] + if claims.judgeable(step_claim): + covering = [ + e + for e in claims.entries_for(step_claim, indicator) + if claims.covers(e, item_id, indicator) + ] + again = claims.verdict(kind, before, after, way, covering) + if again != said: + return [ + f"{where}, {indicator}, {steps[k]['id']}: written {said!r}, the claim gives {again!r}" + ] + block["verdicts"][said] += 1 + worth = worth or said == claims.NOT_MOVED + # attribution, as `chain.build` makes it + one = attributed[indicator] + if any(not s["explained"] and kinds[j] in rule.MOVES for j, s in enumerate(steps)): + one["unexplained_moved"] += 1 + if kinds[nsteps] in rule.MOVES: + if any(v is None for v in values): + one["not_attributable"] += 1 + else: + carrier = rule.largest_step(values, kinds[:nsteps]) + if carrier is None: + one["spread"] += 1 + else: + one["dominant"][steps[carrier]["id"]] += 1 + qualifying += worth + if item_id in listed: + item = listed[item_id] + if [item["name"], item["place"], item["group"], item["unit"]] != [name, place, group, unit]: + found.append(f"{where}: its name, place, group or unit differ from the diagnostics file") + if item["values"] != values_of: + found.append(f"{where}: its results differ from the diagnostics file") + missing_listed = set(listed) - {row[0] for row in m["items"]} + if missing_listed: + found.append(f"{len(missing_listed)} listed items are not in the matrix") + if data["items_count"]["qualifying"] != qualifying: + found.append( + f"the diagnostics file counts {data['items_count']['qualifying']} items worth a look, " + f"the matrix holds {qualifying}" + ) + for (k, indicator), block in counted.items(): + name = fileformat.TOTAL if k is None else steps[k]["id"] + written = ( + data["counts"][fileformat.TOTAL][indicator] + if k is None + else data["counts"]["steps"][name][indicator] + ) + for key in block: + if block[key] != written[key]: + found.append(f"counts of {name!r} in {indicator!r}: `{key}` is not what the matrix holds") + for indicator in ids: + if attributed[indicator] != data["attribution"][indicator]: + found.append(f"attribution in {indicator!r} is not what the matrix holds") + return found diff --git a/src/sentier_diagnostics/page.py b/src/sentier_diagnostics/page.py index 3318695..25acf45 100644 --- a/src/sentier_diagnostics/page.py +++ b/src/sentier_diagnostics/page.py @@ -1,69 +1,166 @@ -"""The static page: three files that need no build step, no server and no network. +"""The page: a frameless body a site holds in its own page, or a page of its own. -`write(data, folder)` copies `index.html`, `diagnostics.js` and `diagnostics.css` into a -folder and writes the file beside them as `data.js`, so that the page opens from disk by -a double click (a browser does not let a page read a JSON file from disk by itself). The -breakdowns the file names go into `breakdowns.js` the same way. Served over HTTP, the -page can also read a JSON file named in its address (`index.html?file=diagnostics.json`), -and it can open a file a reader chooses. +`body(data)` is the diagnostics as one element, `
`, with the +file inside it as JSON: no ``, no ``, no id. A site puts it where it wants and +loads `diagnostics.js` and `diagnostics.css` from `assets()`. The style is scoped to that +element and sets no font and nothing on `:root` or `body`; its colours take the site's tokens +(`--ink`, `--line`, `--surface` ...) where the site sets them. + +`write(data, folder)` writes a page of its own: `index.html` with the body in it, the two +assets and `standalone.css`, which sets the tokens and font a site would. It opens from disk +by a double click and needs no server and no network. A local file may name its matrix of +every item (`chain.build_with_matrix`); given it, `write` puts it beside the page as +`matrix.js`, which the page loads after it has drawn. The page shows what the file says. It works out no class, no direction and no count of its own: it only adds the file's counts up (the classes that moved into "moved", a step's -counts over its indicators), and splits one item's difference into the steps' shares. +counts over its indicators), divides an item's result after by its result before to show +their ratio, and splits one item's difference into the steps' shares. """ from __future__ import annotations +import html import json +import re import shutil from collections.abc import Mapping from importlib import resources from pathlib import Path from . import breakdown, fileformat +from . import matrix as matrix_format -ASSETS = ("index.html", "diagnostics.js", "diagnostics.css") +#: What a site copies into its assets to hold the body. +ASSETS = ("diagnostics.js", "diagnostics.css") +#: What a page of its own adds: its frame and the template of the body. +STANDALONE = ("index.html", "standalone.css") +TEMPLATE = "body.html" +MATRIX_SCRIPT = "matrix.js" +_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") +_ROOT = '
' -def as_script(name: str, value) -> str: - """A value as a script that assigns it: ASCII only, and no ` str: + """A value as JSON that no HTML parser reads as markup: ASCII only, and every `<` escaped, + so that no text of a file can end the element that holds it or open a comment.""" text = json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) - text = text.replace(" str: + """A value as a script that assigns it (`window.NAME = ...;`), for a site that keeps the + data in a file of its own; the page reads `SENTIER_DIAGNOSTICS`, + `SENTIER_DIAGNOSTICS_BREAKDOWNS` and `SENTIER_DIAGNOSTICS_MATRIX`.""" + return f"window.{name} = {_json(value)};\n" + + +def _read(name: str) -> str: + return (resources.files(__package__) / "page" / name).read_text(encoding="utf-8") -def assets() -> dict[str, Path]: - """The page's own files by name, for a site that copies them into its assets and - writes `data.js` itself (`as_script("SENTIER_DIAGNOSTICS", data)`). The paths are those - of the installed package.""" +def assets(standalone: bool = False) -> dict[str, Path]: + """The page's own files by name, as paths of the installed package: the script and the + style a site copies, and with `standalone` also the frame of a page of its own and the + template of the body.""" source = resources.files(__package__) / "page" - return {name: Path(str(source / name)) for name in ASSETS} + names = ASSETS + ((*STANDALONE, TEMPLATE) if standalone else ()) + return {name: Path(str(source / name)) for name in names} -def write(data: Mapping, folder: Path, breakdowns: Mapping[str, Mapping] | None = None) -> Path: - """The page and its data in `folder`; returns the path of `index.html`.""" +def _checked(data: Mapping, breakdowns: Mapping[str, Mapping]) -> None: found = fileformat.errors(data) if found: raise ValueError("not a diagnostics file: " + "; ".join(found[:10])) - errors = breakdown_errors(data, breakdowns or {}) + errors = breakdown_errors(data, breakdowns) if errors: raise ValueError("; ".join(errors)) - script = as_script("SENTIER_DIAGNOSTICS", data) - if len(script) > fileformat.MAX_BYTES: # ASCII: one byte per character + + +def body( + data: Mapping, + *, + breakdowns: Mapping[str, Mapping] | None = None, + base: str = "", + title: bool = True, + matrix: str | None = None, +) -> str: + """The diagnostics as one element for a site's page, the file and its breakdowns inside. + + `base` is put before the names of the files the page loads (breakdowns, the matrix), for + a site that keeps them elsewhere than the page. `title=False` leaves the heading to the + site. `matrix` names the file of the matrix the page loads after it has drawn, where it + is not the one the file names (a site that serves `matrix.json` needs none). + """ + return _body(data, breakdowns=breakdowns, base=base, title=title, matrix=matrix, standalone=False) + + +def _body(data, *, breakdowns, base, title, matrix, standalone) -> str: + breakdowns = dict(breakdowns or {}) + _checked(data, breakdowns) + if matrix is not None and not (_NAME.fullmatch(matrix) and ".." not in matrix): + raise ValueError(f"the matrix {matrix!r} is not a plain file name") + scripts = f'\n' + if breakdowns: + scripts += f'\n' + if len(scripts) > fileformat.MAX_BYTES: # ASCII: one byte per character raise ValueError( - f"data.js would be {len(script)} bytes, and a page holds {fileformat.MAX_BYTES} at most: list fewer items" + f"the page would hold {len(scripts)} bytes of data, and holds {fileformat.MAX_BYTES} at most: " + "list fewer items, and keep every item in the matrix" ) + attributes = "" + if base: + attributes += f' data-sd-base="{html.escape(base, quote=True)}"' + if matrix: + attributes += f' data-sd-matrix="{html.escape(matrix, quote=True)}"' + if standalone: + attributes += " data-sd-standalone" + text = _read(TEMPLATE) + if _ROOT not in text or "" not in text: + raise RuntimeError("the template of the body has lost its root or its place for the data") + text = text.replace(_ROOT, _ROOT[:-1] + attributes + ">", 1) + if not title: + text = text.replace( + '

', '

{name}

\n{inner}", 1) folder = Path(folder) folder.mkdir(parents=True, exist_ok=True) source = resources.files(__package__) / "page" - for name in ASSETS: - with resources.as_file(source / name) as path: - shutil.copyfile(path, folder / name) - (folder / "data.js").write_text(script, encoding="utf-8") - (folder / "breakdowns.js").write_text( - as_script("SENTIER_DIAGNOSTICS_BREAKDOWNS", dict(breakdowns or {})), encoding="utf-8" - ) + for one in (*ASSETS, "standalone.css"): + with resources.as_file(source / one) as path: + shutil.copyfile(path, folder / one) + (folder / "index.html").write_text(page, encoding="utf-8") + stale = folder / MATRIX_SCRIPT + if script is not None: + stale.write_text(script, encoding="utf-8") + elif stale.exists(): + stale.unlink() # the matrix of an earlier page: this one names none return folder / "index.html" @@ -100,3 +197,16 @@ def named_breakdowns(data: Mapping, beside: Path) -> tuple[dict, list[str]]: continue found[name] = fileformat.load_json(path) return found, absent + + +def named_matrix(data: Mapping, beside: Path) -> dict | None: + """The matrix a file names, read from beside it and checked against the file; None where + it names none or where the matrix is not there (a page then shows the listed items and + says so). A matrix that is there and not the file's is refused.""" + named = data.get("matrix") + if not named: + return None + path = Path(beside) / named["file"] + if not path.is_file(): + return None + return matrix_format.read(path, data) diff --git a/src/sentier_diagnostics/page/body.html b/src/sentier_diagnostics/page/body.html new file mode 100644 index 0000000..dbeaf20 --- /dev/null +++ b/src/sentier_diagnostics/page/body.html @@ -0,0 +1,106 @@ +
+
+

Diagnostics

+

Diagnostics

+

+
    + + +

    +
    + +
    + + +

    Matrix, distributions and worst show this comparison. The size hides smaller changes; it changes no count.

    +
    + + + + + +

    + +
    diff --git a/src/sentier_diagnostics/page/diagnostics.css b/src/sentier_diagnostics/page/diagnostics.css index 241041c..49e3bed 100644 --- a/src/sentier_diagnostics/page/diagnostics.css +++ b/src/sentier_diagnostics/page/diagnostics.css @@ -1,110 +1,169 @@ -/* sentier-diagnostics page. No font, script or image is loaded from anywhere else. */ -:root { - --ink: #1d232a; - --muted: #5b6672; - --paper: #ffffff; - --paper-deep: #f3f5f7; - --line: #d9dee3; - --up: #1f6fb2; /* went up: blue */ - --down: #b4560f; /* went down: orange; the pair reads for colour-blind readers */ - --noise: #8a939c; - --error: #a3262a; - --radius: 6px; - --gutter: 16px; - font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; - color: var(--ink); - background: var(--paper); +/* sentier-diagnostics: the style of the diagnostics, inside a host's page or on their own. + Every rule is scoped to .sentier-diagnostics: nothing here touches the host's :root, body, + headings or tables outside it, and no font is set (the host's is inherited). Colours and + sizes are custom properties that take the host's tokens where it sets them (the shared + tokens of the Sentier sites) and fall back otherwise. Up, down and noise keep colours of + their own, so that a direction never takes a site's accent. No font, script or image is + loaded from anywhere else. */ +.sentier-diagnostics { + --sd-ink: var(--ink, #1d232a); + --sd-muted: var(--muted, #5b6672); + --sd-surface: var(--surface, #ffffff); + --sd-paper-deep: var(--paper-deep, #f3f5f7); + --sd-line: var(--line, #d9dee3); + --sd-line-strong: var(--line-strong, #b6bfc8); + --sd-accent: var(--index, #1d625e); + --sd-fail: var(--fail, #a3262a); + --sd-radius: var(--radius, 6px); + --sd-gutter: var(--gutter, 16px); + --sd-t-xs: var(--t-xs, 0.75rem); + --sd-t-sm: var(--t-sm, 0.875rem); + --sd-t-md: var(--t-md, 1rem); + --sd-t-lg: var(--t-lg, 1.2rem); + --sd-s2: var(--s2, 0.5rem); + --sd-s3: var(--s3, 0.75rem); + --sd-s4: var(--s4, 1rem); + --sd-s5: var(--s5, 1.5rem); + --sd-s6: var(--s6, 2rem); + --sd-up: #1f6fb2; /* went up: blue */ + --sd-down: #b4560f; /* went down: orange; the pair reads for colour-blind readers */ + --sd-noise: #8a939c; + --sd-drawer: clamp(22rem, 42vw, 52rem); + color: var(--sd-ink); + font-size: var(--sd-t-md); + line-height: 1.45; } @media (prefers-color-scheme: dark) { - :root { - --ink: #e6eaee; - --muted: #9aa5b0; - --paper: #15191d; - --paper-deep: #1f252b; - --line: #36404a; - --up: #6fb0ec; - --down: #f0a065; - --noise: #77818b; - --error: #ff8a8e; - } + .sentier-diagnostics { --sd-up: #6fb0ec; --sd-down: #f0a065; --sd-noise: #77818b; } } -body { margin: 0; background: var(--paper); line-height: 1.45; } -header.top, main, footer { max-width: 1200px; margin: 0 auto; padding: 0 var(--gutter); } -h1 { font-size: 1.6rem; margin: 0.2rem 0; } -h2 { font-size: 1.25rem; margin: 2rem 0 0.5rem; border-bottom: 1px solid var(--line); padding-bottom: 0.25rem; } -h3 { font-size: 1.05rem; margin: 1.25rem 0 0.4rem; } -.eyebrow { text-transform: uppercase; letter-spacing: 0.06em; font-size: 0.75rem; color: var(--muted); margin: 1rem 0 0; } -.lead { margin: 0.3rem 0; } -.quiet { color: var(--muted); font-size: 0.9rem; } -.error { color: var(--error); } -.plain { list-style: none; padding: 0; } -.plain li { margin: 0.5rem 0; } + /* a class that sets `display` would otherwise show an element marked hidden */ -[hidden] { display: none !important; } -.visually-hidden { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); } -code { font-size: 0.9em; } +.sentier-diagnostics [hidden] { display: none !important; } +.sentier-diagnostics .sd-visually-hidden { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); } + +/* the head */ +.sentier-diagnostics .sd-head { margin: 0 0 var(--sd-s4); } +.sentier-diagnostics .sd-eyebrow { margin: var(--sd-s4) 0 0; font-size: var(--sd-t-xs); letter-spacing: 0.06em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-title { margin: 0.2rem 0; } +.sentier-diagnostics .sd-lead { margin: 0.3rem 0; max-width: 60rem; } +.sentier-diagnostics .sd-quiet { color: var(--sd-muted); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-note { max-width: 70rem; color: var(--sd-muted); font-size: var(--sd-t-sm); line-height: 1.5; } +.sentier-diagnostics .sd-error { color: var(--sd-fail); } +.sentier-diagnostics .sd-plain { list-style: none; padding: 0; } +.sentier-diagnostics .sd-plain > li { margin: var(--sd-s2) 0; } +.sentier-diagnostics .sd-states { display: flex; flex-wrap: wrap; gap: 0.25rem 0.5rem; margin: 0.6rem 0; padding: 0; list-style: none; } +.sentier-diagnostics .sd-states > li { padding: 0.15rem 0.5rem; border: 1px solid var(--sd-line); border-radius: var(--sd-radius); background: var(--sd-paper-deep); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-states > li + li::before { content: "→ "; color: var(--sd-muted); } +.sentier-diagnostics .sd-banner { display: inline-block; margin: var(--sd-s2) 0; padding: 0.3rem 0.6rem; border: 1px solid var(--sd-fail); border-radius: var(--sd-radius); color: var(--sd-fail); } +.sentier-diagnostics .sd-made-by { margin-top: var(--sd-s6); } -.states { display: flex; flex-wrap: wrap; gap: 0.25rem 0.5rem; padding: 0; margin: 0.6rem 0; list-style: none; } -.states li { background: var(--paper-deep); border: 1px solid var(--line); border-radius: var(--radius); padding: 0.15rem 0.5rem; font-size: 0.9rem; } -.states li + li::before { content: "→ "; color: var(--muted); } +/* the bar of the comparison and the size: it stays in view */ +.sentier-diagnostics .sd-bar { position: sticky; top: 0; z-index: 4; display: flex; flex-wrap: wrap; align-items: flex-end; gap: var(--sd-s2) var(--sd-s5); + padding: var(--sd-s2) 0; border-bottom: 1px solid var(--sd-line); background: var(--sd-surface); } +.sentier-diagnostics .sd-bar-note { flex: 1 1 20rem; margin: 0; padding-bottom: 0.4rem; } +.sentier-diagnostics .sd-section { margin-top: var(--sd-s6); } +.sentier-diagnostics .sd-section > h3 { margin: 0 0 var(--sd-s2); padding-bottom: 0.25rem; border-bottom: 1px solid var(--sd-line); } +.sentier-diagnostics h4 { margin: var(--sd-s5) 0 var(--sd-s2); } -.toc { position: sticky; top: 0; z-index: 2; background: var(--paper); border-bottom: 1px solid var(--line); - display: flex; flex-wrap: wrap; gap: 0.4rem 1rem; align-items: center; padding: 0.4rem var(--gutter); - max-width: 1200px; margin: 0 auto; } -.toc a { color: var(--ink); } -.compare { margin-left: auto; font-size: 0.9rem; } -.compare-size { font-size: 0.9rem; } -.banner { border: 1px solid var(--error); color: var(--error); border-radius: var(--radius); padding: 0.3rem 0.6rem; display: inline-block; } -select, input, button { font: inherit; color: inherit; background: var(--paper); border: 1px solid var(--line); border-radius: var(--radius); padding: 0.15rem 0.35rem; } -button { cursor: pointer; } -.controls { display: flex; flex-wrap: wrap; gap: 0.5rem 1rem; margin: 0.5rem 0; font-size: 0.9rem; } +/* fields, buttons */ +.sentier-diagnostics .sd-controls { display: flex; flex-wrap: wrap; align-items: flex-end; gap: var(--sd-s2) var(--sd-s4); margin: var(--sd-s3) 0; } +.sentier-diagnostics .sd-field { display: flex; flex-direction: column; gap: 0.2rem; font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-field > span { font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-grow { flex: 0 1 22rem; } +.sentier-diagnostics select, .sentier-diagnostics input, .sentier-diagnostics .sd-button { + min-height: 2.25rem; padding: 0.25rem 0.5rem; border: 1px solid var(--sd-line-strong); border-radius: var(--sd-radius); + background: var(--sd-surface); color: var(--sd-ink); font: inherit; } +.sentier-diagnostics input[type="search"] { width: 100%; } +.sentier-diagnostics .sd-button { cursor: pointer; font-weight: 700; } +.sentier-diagnostics .sd-button:hover { border-color: var(--sd-accent); color: var(--sd-accent); } +.sentier-diagnostics select:focus-visible, .sentier-diagnostics input:focus-visible, .sentier-diagnostics .sd-button:focus-visible { outline: 2px solid var(--sd-accent); outline-offset: 1px; } -.table-wrap { overflow-x: auto; max-width: 100%; } -table { border-collapse: collapse; font-size: 0.88rem; } -th, td { border-bottom: 1px solid var(--line); padding: 0.25rem 0.5rem; text-align: left; vertical-align: top; } -th { font-weight: 600; background: var(--paper-deep); } -td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } -tr.clickable { cursor: pointer; } -tr.clickable:hover td, tr.clickable:focus td { background: var(--paper-deep); } -.zero { color: var(--muted); } +/* tables: tables at every width, whatever the host does with bare ones */ +.sentier-diagnostics .sd-wrap { max-width: 100%; overflow-x: auto; } +.sentier-diagnostics table { display: table; width: auto; margin: 0; border: 0; border-collapse: collapse; background: none; font-size: var(--sd-t-sm); } +.sentier-diagnostics thead { display: table-header-group; position: static; width: auto; height: auto; overflow: visible; clip-path: none; } +.sentier-diagnostics tbody { display: table-row-group; } +.sentier-diagnostics tr { display: table-row; padding: 0; border: 0; } +.sentier-diagnostics th, .sentier-diagnostics td { display: table-cell; padding: 0.25rem 0.5rem; border: 0; border-bottom: 1px solid var(--sd-line); border-radius: 0; + text-align: left; vertical-align: top; letter-spacing: 0; text-transform: none; } +.sentier-diagnostics td + td::before { content: none; } +.sentier-diagnostics th { font-weight: 700; background: var(--sd-paper-deep); } +.sentier-diagnostics td.num, .sentier-diagnostics th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +.sentier-diagnostics tr.sd-row { cursor: pointer; } +.sentier-diagnostics tr.sd-row:hover > *, .sentier-diagnostics tr.sd-row:focus-visible > * { background: var(--sd-paper-deep); } +.sentier-diagnostics tr.sd-row:focus { outline: none; } +.sentier-diagnostics tr.sd-row:focus-visible > th { box-shadow: inset 3px 0 0 var(--sd-accent); } +.sentier-diagnostics .zero { color: var(--sd-muted); } -/* a value: colour from its direction only, strength from its band */ -.cell { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } -.tone-up { background: color-mix(in srgb, var(--up) calc(var(--s, 0.5) * 55%), transparent); } -.tone-down { background: color-mix(in srgb, var(--down) calc(var(--s, 0.5) * 55%), transparent); } -.tone-up.no-ratio, .tone-down.no-ratio { background-image: repeating-linear-gradient(135deg, transparent 0 4px, rgba(127,127,127,0.35) 4px 6px); } -.tone-noise { color: var(--noise); background-image: repeating-linear-gradient(135deg, transparent 0 5px, rgba(127,127,127,0.18) 5px 7px); } -.tone-near { color: var(--muted); } -.tone-missing { color: var(--error); font-style: italic; } -.tone-none { color: var(--muted); } -.mark { font-size: 0.75em; margin-left: 0.2em; } +/* the matrix: its own frame, scrolling both ways; the names stay on the left, the headings on top */ +.sentier-diagnostics .sd-status { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 0.2rem var(--sd-s5); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-count { margin: 0; font-weight: 700; font-variant-numeric: tabular-nums; } +.sentier-diagnostics .sd-grid-wrap { position: relative; max-height: max(24rem, calc(100vh - 6rem)); overflow: auto; border: 1px solid var(--sd-line); border-radius: var(--sd-radius); } +.sentier-diagnostics table.sd-grid { border-collapse: separate; border-spacing: 0; font-size: var(--sd-t-xs); line-height: 1.35; } +.sentier-diagnostics .sd-grid th, .sentier-diagnostics .sd-grid td { padding: 0.3rem 0.4rem; vertical-align: middle; } +.sentier-diagnostics .sd-grid thead th { position: sticky; top: 0; z-index: 2; vertical-align: bottom; border-bottom-color: var(--sd-line-strong); white-space: normal; min-width: 4.5rem; } +.sentier-diagnostics .sd-grid thead th.sd-name { left: 0; z-index: 3; min-width: 14rem; } +.sentier-diagnostics .sd-grid tbody th { position: sticky; left: 0; z-index: 1; max-width: 18rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; background: var(--sd-surface); font-weight: 600; } +.sentier-diagnostics .sd-grid tbody th .sd-listed { display: inline-block; width: 0.45rem; height: 0.45rem; margin-right: 0.35rem; border-radius: 50%; background: var(--sd-accent); vertical-align: 0.1em; } +.sentier-diagnostics .sd-grid td.sd-group { color: var(--sd-muted); white-space: nowrap; } +.sentier-diagnostics .sd-sort { display: block; width: 100%; padding: 0; border: 0; background: none; color: inherit; font: inherit; font-weight: 700; text-align: right; cursor: pointer; } +.sentier-diagnostics .sd-name .sd-sort { text-align: left; } +.sentier-diagnostics .sd-sort:hover { color: var(--sd-accent); } +.sentier-diagnostics th[aria-sort] { box-shadow: inset 0 -3px 0 var(--sd-accent); } +.sentier-diagnostics tr[aria-current="true"] > th { box-shadow: inset 4px 0 0 var(--sd-accent); } +.sentier-diagnostics .sd-more { position: sticky; left: 0; margin: 0; padding: var(--sd-s3) var(--sd-s4); } -.legend { display: flex; flex-wrap: wrap; gap: 0.3rem 0.9rem; font-size: 0.82rem; margin: 0.4rem 0; } -.legend span.key { display: inline-block; min-width: 2.8em; text-align: center; padding: 0 0.3em; border: 1px solid var(--line); border-radius: 3px; margin-right: 0.3em; } +/* a value: colour from its direction only, strength from how far the two lie apart */ +.sentier-diagnostics .cell { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +.sentier-diagnostics .tone-up { background: color-mix(in srgb, var(--sd-up) calc(var(--s, 0.5) * 50%), var(--sd-surface)); } +.sentier-diagnostics .tone-down { background: color-mix(in srgb, var(--sd-down) calc(var(--s, 0.5) * 50%), var(--sd-surface)); } +.sentier-diagnostics .tone-up.no-ratio, .sentier-diagnostics .tone-down.no-ratio { background-image: repeating-linear-gradient(135deg, transparent 0 4px, rgba(127, 127, 127, 0.35) 4px 6px); font-weight: 700; } +.sentier-diagnostics .tone-noise { color: var(--sd-noise); background-image: repeating-linear-gradient(135deg, transparent 0 5px, rgba(127, 127, 127, 0.18) 5px 7px); } +.sentier-diagnostics .tone-near { color: var(--sd-muted); } +.sentier-diagnostics .tone-missing { color: var(--sd-fail); font-style: italic; } +.sentier-diagnostics .tone-none { color: var(--sd-muted); } +.sentier-diagnostics .mark { margin-left: 0.2em; font-size: 0.75em; } +.sentier-diagnostics .verdict-look { font-weight: 700; } -.bar-cell { min-width: 2.6rem; } -.bar { height: 2.2rem; display: flex; align-items: flex-end; } -.bar span { display: block; width: 100%; background: var(--muted); min-height: 0; } -.bar.up span { background: var(--up); } -.bar.down span { background: var(--down); } -.bar-n { font-size: 0.75rem; text-align: center; color: var(--muted); } +.sentier-diagnostics .sd-legend { display: flex; flex-wrap: wrap; gap: 0.2rem 0.9rem; margin: 0.4rem 0; font-size: var(--sd-t-xs); color: var(--sd-muted); } +.sentier-diagnostics .sd-status > * { min-width: 0; } +.sentier-diagnostics .sd-legend > span { min-width: 0; overflow-wrap: anywhere; } +.sentier-diagnostics .sd-legend .key { display: inline-block; white-space: nowrap; min-width: 2.6em; margin-right: 0.3em; padding: 0 0.3em; border: 1px solid var(--sd-line); border-radius: 3px; text-align: center; color: var(--sd-ink); } -.badge { display: inline-block; font-size: 0.75rem; border: 1px solid var(--line); border-radius: 999px; padding: 0 0.5em; margin-right: 0.4em; } -.badge.model { border-color: var(--down); } -.badge.person { border-color: var(--up); } -.verdict-look { font-weight: 600; } +/* distributions */ +.sentier-diagnostics .bar-cell { min-width: 2.6rem; } +.sentier-diagnostics .bar { display: flex; align-items: flex-end; height: 2.2rem; } +.sentier-diagnostics .bar span { display: block; width: 100%; min-height: 0; background: var(--sd-muted); } +.sentier-diagnostics .bar.up span { background: var(--sd-up); } +.sentier-diagnostics .bar.down span { background: var(--sd-down); } +.sentier-diagnostics .bar-n { font-size: var(--sd-t-xs); text-align: center; color: var(--sd-muted); } -.steps-chart { display: grid; grid-template-columns: minmax(8rem, 14rem) 1fr; gap: 0.15rem 0.6rem; align-items: center; font-size: 0.85rem; margin: 0.4rem 0 1rem; } -.track { position: relative; height: 0.9rem; background: var(--paper-deep); border-radius: 2px; } -.track .seg { position: absolute; top: 0; bottom: 0; } -.track .seg.up { background: var(--up); } -.track .seg.down { background: var(--down); } -.track .axis { position: absolute; top: -2px; bottom: -2px; width: 1px; background: var(--ink); } +/* who wrote what */ +.sentier-diagnostics .badge { display: inline-block; margin-right: 0.4em; padding: 0 0.5em; border: 1px solid var(--sd-line); border-radius: 999px; font-size: var(--sd-t-xs); } +.sentier-diagnostics .badge.model { border-color: var(--sd-down); } +.sentier-diagnostics .badge.person { border-color: var(--sd-up); } +.sentier-diagnostics .sd-scope { display: inline-block; margin-right: 0.3em; padding: 0 0.4em; border: 1px solid var(--sd-line-strong); border-radius: 3px; background: var(--sd-paper-deep); font-size: var(--sd-t-xs); font-weight: 700; } -#item-close { float: right; margin-top: -2.4rem; } -#item { border: 1px solid var(--line); border-radius: var(--radius); padding: 0 var(--gutter) var(--gutter); margin-top: 1.5rem; } +/* the item: a drawer on the right, the whole window on a phone */ +.sentier-diagnostics .sd-drawer { position: fixed; top: 0; right: 0; bottom: 0; z-index: 20; display: flex; flex-direction: column; width: var(--sd-drawer); max-width: 100%; + border-left: 1px solid var(--sd-line-strong); background: var(--sd-surface); box-shadow: -0.5rem 0 2rem color-mix(in srgb, var(--sd-ink) 16%, transparent); } +.sentier-diagnostics .sd-drawer-head { flex: none; display: grid; grid-template-columns: minmax(0, 1fr) auto; column-gap: var(--sd-s4); align-items: start; + padding: var(--sd-s4) var(--sd-s5); border-bottom: 1px solid var(--sd-line); background: var(--sd-paper-deep); } +.sentier-diagnostics .sd-drawer-head > .sd-label { grid-column: 1; margin: 0; font-size: var(--sd-t-xs); font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--sd-muted); } +.sentier-diagnostics .sd-drawer-head > h3 { grid-column: 1; margin: 0.2rem 0 0; font-size: var(--sd-t-lg); line-height: 1.3; overflow-wrap: anywhere; } +.sentier-diagnostics .sd-drawer-head > h3:focus { outline: none; } +.sentier-diagnostics .sd-drawer-head > .sd-button { grid-column: 2; grid-row: 1 / span 2; align-self: center; } +.sentier-diagnostics .sd-drawer-body { flex: 1 1 auto; min-height: 0; overflow: auto; overscroll-behavior: contain; padding: var(--sd-s4) var(--sd-s5) var(--sd-s6); } +.sentier-diagnostics .sd-drawer-body > h4:first-child { margin-top: 0; } +.sentier-diagnostics .sd-why { padding: 0; list-style: none; } +.sentier-diagnostics .sd-why > li { margin: var(--sd-s2) 0; padding: var(--sd-s2) var(--sd-s3); border-left: 3px solid var(--sd-line-strong); border-radius: 0 4px 4px 0; background: var(--sd-paper-deep); font-size: var(--sd-t-sm); } +.sentier-diagnostics .sd-why > li.sd-none { border-left-color: var(--sd-line); background: none; color: var(--sd-muted); } -@media (max-width: 640px) { - .compare { margin-left: 0; } - h1 { font-size: 1.3rem; } +@media (max-width: 47.99rem) { + .sentier-diagnostics { --sd-drawer: 100vw; } + .sentier-diagnostics .sd-drawer { border-left: 0; box-shadow: none; } + .sentier-diagnostics .sd-drawer-head { padding: var(--sd-s3) var(--sd-s4); } + .sentier-diagnostics .sd-drawer-body { padding: var(--sd-s3) var(--sd-s4) var(--sd-s6); } + .sentier-diagnostics .sd-grid tbody th { max-width: 11rem; } + .sentier-diagnostics .sd-grid .sd-group { display: none; } } diff --git a/src/sentier_diagnostics/page/diagnostics.js b/src/sentier_diagnostics/page/diagnostics.js index 6dd36e3..c253e7a 100644 --- a/src/sentier_diagnostics/page/diagnostics.js +++ b/src/sentier_diagnostics/page/diagnostics.js @@ -1,14 +1,25 @@ /* sentier-diagnostics page: shows a diagnostics file (sentier-diagnostics/1) as it is. * - * Plain JavaScript, no library, no network but the file itself. The page works out no - * class, direction, band, verdict or count: they are in the file, checked by the - * validator. It only orders and filters, formats numbers, and subtracts consecutive - * results to show each step's share. Every text of the file is set as text, never as markup. + * Plain JavaScript, no library, no network but the files the page names. The script mounts + * into every element marked `data-sentier-diagnostics` (the body `page.body` writes), on a + * page of its own or inside a host's page; it looks nothing up outside that element but the + * address and the globals named below. A host that draws the element later calls + * `SentierDiagnostics.mount(element, {data})`, and marks it `data-sentier-diagnostics="manual"` + * so that it is not mounted before. + * + * The page works out no class, direction, verdict or count: they are in the file, checked by + * the validator. It only orders and filters, formats numbers, divides the result after by the + * result before to show their ratio, and subtracts consecutive results to show each step's + * share. Every text of a file is set as text, never as markup. */ -(function (root) { +(function (global) { 'use strict'; var FORMAT = 'sentier-diagnostics/1'; + var MATRIX_FORMAT = 'sentier-diagnostics-matrix/1'; + var MATRIX_MAX_ITEMS = 200000; // matrix.MAX_ITEMS + var MATRIX_MAX_BYTES = 64 * 1024 * 1024; // matrix.MAX_BYTES + var PAGE = 300; // rows of the grid drawn at a time var MOVES = ['differs', 'beyond 10x', 'zero reference', 'zero result', 'sign change']; var QUIET = ['agrees', 'within noise', 'near zero']; // the order of the classes from the worst: the same as the Python rule (rule.RANK) @@ -17,6 +28,8 @@ 'missing': 5, 'within noise': 6, 'agrees': 7, 'near zero': 8 }; var STRENGTH = { '1-1.1': 0.3, '1.1-1.25': 0.5, '1.25-2': 0.7, '2-10': 0.85, 'over 10': 1 }; + // the ends of the bands, as rule.SIZE_STEPS: the distance (|log10| of the factor) at which each ends + var SIZE_STEPS = [[Math.log10(1.1), '1-1.1'], [Math.log10(1.25), '1.1-1.25'], [Math.log10(2), '1.25-2'], [1, '2-10']]; var LOOK = ['off claim', 'not claimed', 'claimed, not moved']; var SAID = { 'agrees': 'the two results differ by no more than the tolerance', @@ -33,10 +46,22 @@ 'agrees': '=', 'within noise': '~', 'near zero': '≈ 0', 'missing': '∅', 'zero reference': '0 →', 'zero result': '→ 0', 'sign change': '±' }; + // the codes of a matrix (matrix.CODES, matrix.VERDICT_CODES) + var CODES = { + '=': ['agrees', null], '~': ['within noise', null], '0': ['near zero', null], '-': ['missing', null], + 'D': ['differs', 'up'], 'd': ['differs', 'down'], 'B': ['beyond 10x', 'up'], 'b': ['beyond 10x', 'down'], + 'R': ['zero reference', 'up'], 'r': ['zero reference', 'down'], 'T': ['zero result', 'up'], 't': ['zero result', 'down'], + 'S': ['sign change', 'up'], 's': ['sign change', 'down'] + }; + var VERDICT_CODES = { + 'a': 'as claimed', 'o': 'off claim', 'n': 'not claimed', 'c': 'claimed, not moved', + '~': 'within noise', '=': 'unchanged', '?': 'not judged' + }; + var NO_CLAIM = '.'; // ------------------------------------------------------------------ pure helpers - function groupThousands(n) { return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ' '); } + function groupThousands(n) { return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ' '); } // a result as a reader reads it: four significant digits; never "0" for a value that is not 0 function formatNumber(v) { @@ -45,7 +70,7 @@ var a = Math.abs(v); // one style from 1e4 on: toPrecision(4) would write 12345 as "1.235e+4" itself if (a >= 1e4 || a < 1e-3) { - return v.toExponential(3).replace(/\.?0+e/, 'e').replace('e+', 'e').replace(/-/g, '\u2212'); + return v.toExponential(3).replace(/\.?0+e/, 'e').replace('e+', 'e').replace(/-/g, '−'); } var s = v.toPrecision(4); if (s.indexOf('.') >= 0) s = s.replace(/0+$/, '').replace(/\.$/, ''); @@ -68,7 +93,20 @@ function shareText(part) { var v = part * 100, size = Math.abs(v); var text = size === 0 ? '0' : size < 0.1 ? size.toExponential(1) : size.toFixed(1); - return (v > 0 ? '' : v < 0 ? '\u2212' : '') + text + ' %'; + return (v > 0 ? '' : v < 0 ? '−' : '') + text + ' %'; + } + + // the result after divided by the result before, of two results of one sign; never "1" for + // a ratio that is not 1, and never "Infinity" where the quotient leaves the floats + function ratioText(before, after) { + var r = after / before; + if (!isFinite(r) || r === 0) { + var orders = Math.log10(Math.abs(after)) - Math.log10(Math.abs(before)); + return '1e' + String(Math.round(orders)).replace('-', '−'); + } + var d = r - 1; + if (d !== 0 && Math.abs(d) < 1e-3) return '1 ' + (d > 0 ? '+ ' : '− ') + formatNumber(Math.abs(d)); + return formatNumber(r); } // Steps whose differences add up to more than this many times the whole difference @@ -136,12 +174,37 @@ return typeof value === 'string' && /^https:\/\/[^\s]+$/.test(value) ? value : null; } + // a file a page may load beside itself: a plain relative name, never an address or a way up + function safeName(value) { + return typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/.test(value) && value.indexOf('..') < 0; + } + function bandStep(band) { if (!band) return null; var at = band.indexOf('factor '); return at < 0 ? null : band.slice(at + 7); } + // how far the sizes of two results of one sign lie apart, as rule.distance: the |log10| + // of their factor, or, where the factor is too large or too small for a float, the + // difference of their logarithms, which cannot overflow + var SMALLEST_NORMAL = 2.2250738585072014e-308; + function distance(before, after) { + var factor = Math.abs(after) / Math.abs(before); + if (isFinite(factor) && factor >= SMALLEST_NORMAL) return Math.abs(Math.log10(factor)); + return Math.abs(Math.log10(Math.abs(after)) - Math.log10(Math.abs(before))); + } + + // the band of a pair, as rule.band, for a matrix, which holds no bands: the class and + // direction are the file's, the band only names how far apart the two lie + function bandOf(before, after, cls, dir) { + if (cls === 'beyond 10x') return dir + ', factor over 10'; + if (cls !== 'differs') return null; + var found = distance(before, after); + for (var i = 0; i < SIZE_STEPS.length; i++) if (found <= SIZE_STEPS[i][0]) return dir + ', factor ' + SIZE_STEPS[i][1]; + return dir + ', factor ' + SIZE_STEPS[SIZE_STEPS.length - 1][1]; + } + // the reader's own size filter: it hides values, it changes no class and no count. // A value without a ratio (a zero or a sign change) and a missing one always pass. function sizeAtLeast(cls, band, threshold) { @@ -191,14 +254,16 @@ return view; } - // how far the sizes of two results of one sign lie apart, as rule.distance: the |log10| - // of their factor, or, where the factor is too large or too small for a float, the - // difference of their logarithms, which cannot overflow - var SMALLEST_NORMAL = 2.2250738585072014e-308; - function distance(before, after) { - var factor = Math.abs(after) / Math.abs(before); - if (isFinite(factor) && factor >= SMALLEST_NORMAL) return Math.abs(Math.log10(factor)); - return Math.abs(Math.log10(Math.abs(after)) - Math.log10(Math.abs(before))); + // a value of the matrix grid: the ratio after ÷ before where the two share a sign, else + // what cellView says. Its colour is its direction's, its strength the band's; the ratio + // is a display, the class stays the file's. + function ratioView(before, after, cls, dir, band) { + var view = cellView(before, after, cls, dir, band); + if (cls === 'differs' || cls === 'beyond 10x' || cls === 'within noise') { + view.text = (cls === 'within noise' ? '~ ' : '') + ratioText(before, after); + view.title += ' After ÷ before: ' + String(after / before) + '.'; + } + return view; } // the key that orders values from the worst: the same as the Python rule (rule.severity) @@ -250,419 +315,576 @@ return null; } - // ------------------------------------------------------------------ the page - - var D = null, B = {}; - var view = { comp: null, worstN: 20, worstIndicator: '', find: '', cls: '', verdict: '', group: '', limit: 100, size: 0 }; - - function el(tag, props, children) { - var node = document.createElement(tag); - if (props) { - Object.keys(props).forEach(function (key) { - var value = props[key]; - if (value === null || value === undefined || value === false) return; - if (key === 'text') node.textContent = value; - else if (key === 'cls') node.className = value; - else if (key === 'style') node.setAttribute('style', value); - else node.setAttribute(key, value === true ? '' : value); - }); - } - (children || []).forEach(function (child) { - if (child === null || child === undefined) return; - node.appendChild(typeof child === 'string' ? document.createTextNode(child) : child); - }); - return node; + function sameList(a, b) { + if (!Array.isArray(a) || a.length !== b.length) return false; + for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; } - function q(id) { return document.getElementById(id); } - function clear(node) { while (node.firstChild) node.removeChild(node.firstChild); return node; } - function n(x) { return x === null || x === undefined ? '—' : groupThousands(x); } - function indicatorName(id) { - for (var i = 0; i < D.indicators.length; i++) { - if (D.indicators[i].id === id) return D.indicators[i].short || D.indicators[i].name; + // what a page needs of a matrix before it draws a row of it: that it is of the file the + // page shows, and that every row has the shape the format gives. Its codes are not worked + // out again here: `sentier-diagnostics check` does that. + function checkMatrix(m, d) { + if (!m || typeof m !== 'object') return 'not a matrix'; + if (m.format !== MATRIX_FORMAT) return 'not a matrix of format ' + MATRIX_FORMAT; + var of = m.of || {}; + var ids = d.indicators.map(function (i) { return i.id; }); + if (of.title !== d.title || !sameList(of.states, d.states.map(function (s) { return s.id; })) || + !sameList(of.indicators, ids) || !sameList(of.steps, d.steps.map(function (s) { return s.id; }))) { + return 'it is the matrix of another file: its title, states, indicators or steps differ'; } - return id; - } - function stepIndex(comp) { // the position of a comparison in an item's lists of classes - if (comp === 'total') return D.steps.length; - for (var i = 0; i < D.steps.length; i++) if (D.steps[i].id === comp) return i; - return D.steps.length; - } - function compLabel(comp) { - if (comp === 'total') return 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; - var k = stepIndex(comp); - return 'Step ' + k + ': ' + D.steps[k].label; - } - function pairOf(item, indicator, k) { - var values = item.values[indicator]; - return k === D.steps.length ? [values[0], values[values.length - 1]] : [values[k], values[k + 1]]; - } - function verdictOf(item, indicator, k) { - if (!item.verdicts || k >= D.steps.length) return null; - return item.verdicts[indicator][k]; - } - - function cell(item, indicator, k) { - var pair = pairOf(item, indicator, k); - var cls = item.classes[indicator][k]; - var v = cellView(pair[0], pair[1], cls, item.directions[indicator][k], item.bands[indicator][k]); - var verdict = verdictOf(item, indicator, k); - var title = v.title + (verdict ? ' Verdict: ' + verdict + '.' : ''); - var classes = 'cell tone-' + v.tone + (v.noRatio ? ' no-ratio' : ''); - var td = el('td', { cls: classes, title: title, style: v.strength ? '--s:' + v.strength : null, 'data-class': cls }, - [v.text, v.mark ? el('span', { cls: 'mark', text: v.mark, 'aria-hidden': 'true' }) : null]); - if (verdict && LOOK.indexOf(verdict) >= 0) td.appendChild(el('span', { cls: 'mark verdict-look', text: verdict === 'claimed, not moved' ? '○' : verdict === 'off claim' ? '◐' : '■', title: verdict })); - return td; + if (!Array.isArray(m.items) || m.items.length > MATRIX_MAX_ITEMS) return 'its items are not a list of at most ' + groupThousands(MATRIX_MAX_ITEMS); + var n = m.items.length; + if (d.matrix && d.matrix.items !== n) return 'the file says ' + d.matrix.items + ' items, the matrix holds ' + n; + var claimed = d.steps.some(function (s) { return !!s.claim; }); + if (!Array.isArray(m.values) || m.values.length !== n || !Array.isArray(m.classes) || m.classes.length !== n) return 'its values or classes are not one row per item'; + if (claimed ? !Array.isArray(m.verdicts) || m.verdicts.length !== n : m.verdicts !== null) return 'its verdicts are not one row per item, or null where no step has a claim'; + var ns = d.states.length, nk = d.steps.length, ni = ids.length; + for (var at = 0; at < n; at++) { + var row = m.items[at]; + if (!Array.isArray(row) || row.length !== 5 || typeof row[0] !== 'string' || typeof row[1] !== 'string') return 'item ' + at + ' is not [id, name, place, group, unit]'; + var values = m.values[at]; + if (!Array.isArray(values) || values.length !== ni * ns) return 'item ' + row[0] + ': not one result per indicator and state'; + for (var v = 0; v < values.length; v++) if (values[v] !== null && (typeof values[v] !== 'number' || !isFinite(values[v]))) return 'item ' + row[0] + ': a result that is not a number'; + var codes = m.classes[at]; + if (typeof codes !== 'string' || codes.length !== ni * (nk + 1)) return 'item ' + row[0] + ': not one class code per indicator and comparison'; + for (var c = 0; c < codes.length; c++) if (!CODES.hasOwnProperty(codes[c])) return 'item ' + row[0] + ': an unknown class code'; + if (claimed) { + var said = m.verdicts[at]; + if (typeof said !== 'string' || said.length !== ni * nk) return 'item ' + row[0] + ': not one verdict code per indicator and step'; + for (var s = 0; s < said.length; s++) if (said[s] !== NO_CLAIM && !VERDICT_CODES.hasOwnProperty(said[s])) return 'item ' + row[0] + ': an unknown verdict code'; + } + } + return null; } - function legend(node) { - clear(node); - var keys = [ - ['tone-up', 'up', 'went up (as signed numbers)'], - ['tone-down', 'down', 'went down: two results below zero that go further below zero went down'], - ['tone-up no-ratio', '0 →', 'from zero, to zero or across zero: no ratio'], - ['tone-none', '=', 'agrees within the tolerance'], - ['tone-noise', '~', 'within the measured noise'], - ['tone-near', '≈ 0', 'near zero on both sides'], - ['tone-missing', 'missing', 'a side has no result'] - ]; - keys.forEach(function (k) { - node.appendChild(el('span', null, [el('span', { cls: 'key ' + k[0], style: '--s:0.7', text: k[1] }), k[2]])); + // the rows of a checked matrix, in the shape of a listed item: values, classes, directions + // and verdicts per indicator. A listed item keeps its own record beside the row. + function matrixRows(m, d) { + var ids = d.indicators.map(function (i) { return i.id; }); + var ns = d.states.length, nk = d.steps.length; + var listed = {}; + d.items.forEach(function (item) { listed[item.id] = item; }); + return m.items.map(function (row, at) { + var out = { id: row[0], name: row[1], place: row[2], group: row[3], unit: row[4], values: {}, classes: {}, directions: {}, + bands: null, verdicts: m.verdicts ? {} : null, item: listed[row[0]] || null }; + ids.forEach(function (id, x) { + out.values[id] = m.values[at].slice(x * ns, (x + 1) * ns); + var codes = m.classes[at].slice(x * (nk + 1), (x + 1) * (nk + 1)).split(''); + out.classes[id] = codes.map(function (c) { return CODES[c][0]; }); + out.directions[id] = codes.map(function (c) { return CODES[c][1]; }); + if (m.verdicts) { + out.verdicts[id] = m.verdicts[at].slice(x * nk, (x + 1) * nk).split('').map(function (c) { return c === NO_CLAIM ? null : VERDICT_CODES[c]; }); + } + }); + if (out.item) out.bands = out.item.bands; + return out; }); - node.appendChild(el('span', null, ['■ not claimed, ◐ off claim, ○ claimed, not moved'])); } - function renderHeader() { - document.title = D.title; - q('title').textContent = D.title; - q('about').textContent = D.about || ''; - var states = clear(q('states')); - D.states.forEach(function (s) { - var identity = Object.keys(s.identity).map(function (k) { return k + ': ' + s.identity[k]; }).join(', '); - var join = s.join ? n(s.join.compared) + ' compared, ' + n(s.join.unmatched) + ' unmatched, ' + n(s.join.ambiguous) + ' ambiguous' : ''; - states.appendChild(el('li', { title: (s.about || '') + (identity ? ' (' + identity + ')' : '') + (s.join && s.join.note ? ' ' + s.join.note : '') }, - [s.label + (s.kind === 'published' ? ' (published)' : ''), join ? el('span', { cls: 'quiet', text: ' · ' + join }) : null])); - }); - var c = D.items_count; - q('tier').textContent = groupThousands(c.all) + ' ' + D.item.many + ', ' + D.indicators.length + ' indicators. ' + - (D.tier === 'counts' ? 'This file holds counts only: no single ' + D.item.one + ' is shown.' - : groupThousands(c.listed) + ' of the ' + groupThousands(c.qualifying) + ' ' + D.item.many + ' worth a look are in this file' + - (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' are not.' : '.')); - q('audience').textContent = D.audience === 'local' - ? 'Local file: its writer has not said that it may be published.' : ''; - q('audience').hidden = D.audience !== 'local'; - // only a chain that starts from the published results has a step 0 that reproduces them - q('steps-zero').hidden = D.states[0].kind !== 'published'; - q('made-by').textContent = 'Written by ' + D.made_by + '. ' + FORMAT + '.'; - var select = clear(q('comparison')); - select.appendChild(el('option', { value: 'total', text: 'The whole way' })); - D.steps.forEach(function (s, k) { select.appendChild(el('option', { value: s.id, text: 'Step ' + k + ': ' + s.label })); }); - select.value = view.comp; + function listedRow(item) { + return { id: item.id, name: item.name, place: item.place, group: item.group, unit: item.unit, values: item.values, + classes: item.classes, directions: item.directions, bands: item.bands, verdicts: item.verdicts, item: item }; } - function sumOver(comp, fn) { - var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; - return Object.keys(blocks).reduce(function (acc, key) { return acc + fn(blocks[key]); }, 0); - } - function moved(block) { return MOVES.reduce(function (a, k) { return a + block.classes[k]; }, 0); } - - function renderSteps() { - var table = clear(q('steps-table')); - var head = ['#', 'Step', 'Kind', 'Explained', 'Claim', 'Moved', 'Up', 'Down', 'As claimed', 'Off claim', 'Not claimed', 'Claimed, not moved', 'Within noise', 'Missing before', 'Missing after', 'Tolerance', 'Noise']; - table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); - var body = el('tbody'); - D.steps.forEach(function (s, k) { - var verdict = function (name) { return s.claim ? n(sumOver(s.id, function (b) { return b.verdicts[name]; })) : '—'; }; - var noise = noiseText(s.noise, D.indicators.map(function (i) { return i.id; })); - // the step from the published results: what moved there is the residual the chain - // starts from, not a regression, whatever its claim says - var unrepro = sumOver(s.id, moved); - var residual = k === 0 && D.states[0].kind === 'published' - ? (unrepro === 1 ? '1 value' : n(unrepro) + ' values') + ' not reproduced within the tolerance: the residual the chain starts from, not a regression.' : null; - body.appendChild(el('tr', null, [ - el('td', { cls: 'num', text: String(k) }), - el('td', null, [safeLink(s.link) ? el('a', { href: safeLink(s.link), text: s.label }) : s.label, s.about ? el('div', { cls: 'quiet', text: s.about }) : null, - residual ? el('div', { cls: 'quiet', text: residual }) : null]), - el('td', { text: s.kind }), - el('td', { text: s.explained ? 'yes' : 'no: nothing explains it', cls: s.explained ? null : 'error' }), - el('td', null, [s.claim ? s.claim.text : 'no claim', s.claim && s.claim.by ? el('div', null, [badge(s.claim.by)]) : null, - s.claim && s.claim.scope.some(function (e) { return e.items === null; }) - ? el('div', { cls: 'quiet', text: 'Its list of ' + D.item.many + ' is not in this file' + (s.claim.reach !== null ? ' (' + n(s.claim.reach) + ')' : '') + ': the verdicts were worked out by its writer and cannot be checked again here.' }) : null]), - el('td', { cls: 'num', text: n(sumOver(s.id, moved)) }), - el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.up; })) }), - el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.down; })) }), - el('td', { cls: 'num', text: verdict('as claimed') }), - el('td', { cls: 'num', text: verdict('off claim') }), - el('td', { cls: 'num', text: verdict('not claimed') }), - el('td', { cls: 'num', text: verdict('claimed, not moved') }), - el('td', { cls: 'num', text: verdict('within noise') }), - el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.before + b.missing.both; })) }), - el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.after + b.missing.both; })) }), - el('td', { cls: 'num', text: String(s.tolerance.value), title: s.tolerance.basis + (s.tolerance.note ? ': ' + s.tolerance.note : '') }), - el('td', { text: noise }) - ])); - }); - table.appendChild(body); - - var total = clear(q('total-table')); - q('total-heading').textContent = 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; - var classes = Object.keys(D.counts.total[D.indicators[0].id].classes); - var cols = ['Indicator'].concat(classes, ['Up', 'Down', 'Largest step', 'Spread over steps', 'Not attributable', 'Moved by what nothing explains']); - total.appendChild(el('thead', null, [el('tr', null, cols.map(function (h) { return el('th', { scope: 'col', text: h, title: SAID[h] || null }); }))])); - var tbody = el('tbody'); - D.indicators.forEach(function (ind) { - var block = D.counts.total[ind.id]; - var att = D.attribution[ind.id]; - var ranked = Object.keys(att.dominant).filter(function (k) { return att.dominant[k]; }) - .sort(function (a, b) { return att.dominant[b] - att.dominant[a]; }) - .map(function (k) { return D.steps[stepIndex(k)].label + ' (' + att.dominant[k] + ')'; }); - var row = [el('th', { scope: 'row', text: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') })]; - classes.forEach(function (c) { - var title = c === 'missing' ? 'missing before: ' + block.missing.before + ', after: ' + block.missing.after + ', on both sides: ' + block.missing.both : null; - row.push(el('td', { cls: 'num' + (block.classes[c] ? '' : ' zero'), text: n(block.classes[c]), title: title })); + // ------------------------------------------------------------------ one mounted page + + function mount(root, given) { + if (!root || root.getAttribute('data-sd-mounted') !== null) return null; + root.setAttribute('data-sd-mounted', ''); + given = given || {}; + var doc = root.ownerDocument || global.document; + var win = global; + var standalone = root.getAttribute('data-sd-standalone') !== null; + var base = root.getAttribute('data-sd-base') || ''; + + var D = null, B = {}, LISTED = [], M = null; + var matrix = { state: 'none', said: '' }; + var view = { comp: null, worstN: 20, worstIndicator: '', find: '', cls: '', verdict: '', group: '', limit: PAGE, size: 0, sort: null, open: null }; + var lastFocus = null; + + function q(name) { return root.querySelector('[data-sd="' + name + '"]'); } + function el(tag, props, children) { + var node = doc.createElement(tag); + if (props) { + Object.keys(props).forEach(function (key) { + var value = props[key]; + if (value === null || value === undefined || value === false) return; + if (key === 'text') node.textContent = value; + else if (key === 'cls') node.className = value; + else if (key === 'style') node.setAttribute('style', value); + else node.setAttribute(key, value === true ? '' : value); + }); + } + (children || []).forEach(function (child) { + if (child === null || child === undefined) return; + node.appendChild(typeof child === 'string' ? doc.createTextNode(child) : child); }); - row.push(el('td', { cls: 'num', text: n(block.directions.up) }), el('td', { cls: 'num', text: n(block.directions.down) })); - row.push(el('td', { text: ranked.join(', ') || 'none' })); - row.push(el('td', { cls: 'num', text: n(att.spread), title: 'its total moved, and no single step moved it: small steps that add up' })); - row.push(el('td', { cls: 'num', text: n(att.not_attributable), title: 'its total moved, and it has no result in a state between: its difference cannot be split' })); - row.push(el('td', { cls: 'num', text: n(att.unexplained_moved), title: 'moved in a step that no cause explains' })); - tbody.appendChild(el('tr', null, row)); - }); - total.appendChild(tbody); - } + return node; + } + function clear(node) { while (node.firstChild) node.removeChild(node.firstChild); return node; } + function n(x) { return x === null || x === undefined ? '—' : groupThousands(x); } + function many(count) { return count === 1 ? D.item.one : D.item.many; } + function capital(text) { return text.charAt(0).toUpperCase() + text.slice(1); } - function renderDistributions() { - var comp = view.comp; - var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; - var first = blocks[D.indicators[0].id]; - var bands = Object.keys(first.bands); - var down = bands.filter(function (b) { return b.indexOf('down') === 0; }); - var up = bands.filter(function (b) { return b.indexOf('up') === 0; }); - var middle = ['agrees', 'within noise']; - var aside = ['zero reference', 'zero result', 'sign change', 'near zero', 'missing']; - var columns = down.map(function (b) { return ['band', b, 'down']; }) - .concat(middle.map(function (c) { return ['class', c, '']; })) - .concat(up.map(function (b) { return ['band', b, 'up']; })) - .concat(aside.map(function (c) { return ['class', c, '']; })); - var table = clear(q('dist-table')); - var head = [el('th', { scope: 'col', text: 'Indicator' })]; - columns.forEach(function (c) { - var label = c[0] === 'band' ? (c[2] === 'up' ? '↑ ' : '↓ ') + bandStep(c[1]) : (SHORT[c[1]] || c[1]); - head.push(el('th', { scope: 'col', cls: 'num', text: label, title: c[1] + (SAID[c[1]] ? ': ' + SAID[c[1]] : '') })); - }); - table.appendChild(el('thead', null, [el('tr', null, head)])); - var body = el('tbody'); - D.indicators.forEach(function (ind) { - var block = blocks[ind.id]; - var row = [el('th', { scope: 'row', text: indicatorName(ind.id) })]; - columns.forEach(function (c) { - var count = c[0] === 'band' ? block.bands[c[1]] : block.classes[c[1]]; - var share = block.pairs ? count / block.pairs : 0; - var bar = el('div', { cls: 'bar ' + c[2] }, [el('span', { style: 'height:' + (count ? Math.max(1, Math.round(share * 100)) : 0) + '%' })]); - row.push(el('td', { cls: 'bar-cell', title: c[1] + ': ' + count + ' of ' + block.pairs }, [bar, el('div', { cls: 'bar-n', text: groupThousands(count) })])); + function indicatorName(id) { + for (var i = 0; i < D.indicators.length; i++) { + if (D.indicators[i].id === id) return D.indicators[i].short || D.indicators[i].name; + } + return id; + } + function indicatorIds() { return D.indicators.map(function (i) { return i.id; }); } + function stepIndex(comp) { // the position of a comparison in an item's lists of classes + if (comp === 'total') return D.steps.length; + for (var i = 0; i < D.steps.length; i++) if (D.steps[i].id === comp) return i; + return D.steps.length; + } + function stateIndex(stateId) { + for (var i = 0; i < D.states.length; i++) if (D.states[i].id === stateId) return i; + return 0; + } + function compLabel(comp) { + if (comp === 'total') return 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; + var k = stepIndex(comp); + return 'Step ' + k + ': ' + D.steps[k].label; + } + function pairOf(row, indicator, k) { + var values = row.values[indicator]; + return k === D.steps.length ? [values[0], values[values.length - 1]] : [values[k], values[k + 1]]; + } + function verdictOf(row, indicator, k) { + if (!row.verdicts || k >= D.steps.length) return null; + return row.verdicts[indicator][k]; + } + function bandAt(row, indicator, k) { + if (row.bands) return row.bands[indicator][k]; + var pair = pairOf(row, indicator, k); + return bandOf(pair[0], pair[1], row.classes[indicator][k], row.directions[indicator][k]); + } + + // the rows the grid and the worst list draw from: every item once the matrix is loaded, + // the listed items before + function rows() { return M || LISTED; } + function rowById(id) { + var all = rows(); + for (var i = 0; i < all.length; i++) if (all[i].id === id) return all[i]; + return null; + } + function heldSaid() { + var c = D.items_count; + if (M) return 'all ' + groupThousands(M.length) + ' ' + many(M.length) + ' (the matrix)'; + return 'the ' + groupThousands(c.listed) + ' ' + many(c.listed) + ' in this file' + + (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' more worth a look are not in it' : ''); + } + + function valueCell(row, indicator, k, ratio) { + var pair = pairOf(row, indicator, k); + var cls = row.classes[indicator][k]; + var v = (ratio ? ratioView : cellView)(pair[0], pair[1], cls, row.directions[indicator][k], bandAt(row, indicator, k)); + var verdict = verdictOf(row, indicator, k); + var title = v.title + (verdict ? ' Verdict: ' + verdict + '.' : ''); + var classes = 'cell tone-' + v.tone + (v.noRatio ? ' no-ratio' : ''); + var td = el('td', { cls: classes, title: title, style: v.strength ? '--s:' + v.strength : null, 'data-class': cls }, + [v.text, v.mark ? el('span', { cls: 'mark', text: v.mark, 'aria-hidden': 'true' }) : null]); + if (verdict && LOOK.indexOf(verdict) >= 0) td.appendChild(el('span', { cls: 'mark verdict-look', text: verdict === 'claimed, not moved' ? '○' : verdict === 'off claim' ? '◐' : '■', title: verdict })); + return td; + } + + function legend(node, ratio) { + clear(node); + var keys = [ + ['tone-up', ratio ? '1.3' : 'up', 'went up (as signed numbers)'], + ['tone-down', ratio ? '0.77' : 'down', 'went down: two results below zero that go further below zero went down'], + ['tone-up no-ratio', '0 →', 'from zero, to zero or across zero: no ratio'], + ['tone-none', '=', 'agrees within the tolerance'], + ['tone-noise', '~', 'within the measured noise'], + ['tone-near', '≈ 0', 'near zero on both sides'], + ['tone-missing', 'missing', 'a side has no result'] + ]; + keys.forEach(function (k) { + node.appendChild(el('span', null, [el('span', { cls: 'key ' + k[0], style: '--s:0.7', text: k[1] }), k[2]])); }); - body.appendChild(el('tr', null, row)); - }); - table.appendChild(body); - } + node.appendChild(el('span', null, ['■ not claimed, ◐ off claim, ○ claimed, not moved'])); + } + + function renderHeader() { + if (standalone) doc.title = D.title; + q('title').textContent = D.title; + q('about').textContent = D.about || ''; + var states = clear(q('states')); + D.states.forEach(function (s) { + var identity = Object.keys(s.identity).map(function (k) { return k + ': ' + s.identity[k]; }).join(', '); + var join = s.join ? n(s.join.compared) + ' compared, ' + n(s.join.unmatched) + ' unmatched, ' + n(s.join.ambiguous) + ' ambiguous' : ''; + states.appendChild(el('li', { title: (s.about || '') + (identity ? ' (' + identity + ')' : '') + (s.join && s.join.note ? ' ' + s.join.note : '') }, + [s.label + (s.kind === 'published' ? ' (published)' : ''), join ? el('span', { cls: 'sd-quiet', text: ' · ' + join }) : null])); + }); + var c = D.items_count; + q('tier').textContent = groupThousands(c.all) + ' ' + D.item.many + ', ' + D.indicators.length + ' indicators. ' + + (D.tier === 'counts' ? 'This file holds counts only: no single ' + D.item.one + ' is shown.' + : groupThousands(c.listed) + ' of the ' + groupThousands(c.qualifying) + ' ' + D.item.many + ' worth a look are in this file' + + (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' are not.' : '.')); + q('audience').textContent = D.audience === 'local' + ? 'Local file: its writer has not said that it may be published.' : ''; + q('audience').hidden = D.audience !== 'local'; + // only a chain that starts from the published results has a step 0 that reproduces them + q('steps-zero').hidden = D.states[0].kind !== 'published'; + q('made-by').textContent = 'Written by ' + D.made_by + '. ' + FORMAT + '.'; + var select = clear(q('comparison')); + select.appendChild(el('option', { value: 'total', text: 'The whole way' })); + D.steps.forEach(function (s, k) { select.appendChild(el('option', { value: s.id, text: 'Step ' + k + ': ' + s.label })); }); + select.value = view.comp; + } + + function sumOver(comp, fn) { + var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; + return Object.keys(blocks).reduce(function (acc, key) { return acc + fn(blocks[key]); }, 0); + } + function moved(block) { return MOVES.reduce(function (a, k) { return a + block.classes[k]; }, 0); } + + function renderSteps() { + var table = clear(q('steps-table')); + var head = ['#', 'Step', 'Kind', 'Explained', 'Claim', 'Moved', 'Up', 'Down', 'As claimed', 'Off claim', 'Not claimed', 'Claimed, not moved', 'Within noise', 'Missing before', 'Missing after', 'Tolerance', 'Noise']; + table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); + var body = el('tbody'); + D.steps.forEach(function (s, k) { + var verdict = function (name) { return s.claim ? n(sumOver(s.id, function (b) { return b.verdicts[name]; })) : '—'; }; + var noise = noiseText(s.noise, indicatorIds()); + // the step from the published results: what moved there is the residual the chain + // starts from, not a regression, whatever its claim says + var unrepro = sumOver(s.id, moved); + var residual = k === 0 && D.states[0].kind === 'published' + ? (unrepro === 1 ? '1 value' : n(unrepro) + ' values') + ' not reproduced within the tolerance: the residual the chain starts from, not a regression.' : null; + body.appendChild(el('tr', null, [ + el('td', { cls: 'num', text: String(k) }), + el('td', null, [safeLink(s.link) ? el('a', { href: safeLink(s.link), text: s.label }) : s.label, s.about ? el('div', { cls: 'sd-quiet', text: s.about }) : null, + residual ? el('div', { cls: 'sd-quiet', text: residual }) : null]), + el('td', { text: s.kind }), + el('td', { text: s.explained ? 'yes' : 'no: nothing explains it', cls: s.explained ? null : 'sd-error' }), + el('td', null, [s.claim ? s.claim.text : 'no claim', s.claim && s.claim.by ? el('div', null, [badge(s.claim.by)]) : null, + s.claim && s.claim.scope.some(function (e) { return e.items === null; }) + ? el('div', { cls: 'sd-quiet', text: 'Its list of ' + D.item.many + ' is not in this file' + (s.claim.reach !== null ? ' (' + n(s.claim.reach) + ')' : '') + ': the verdicts were worked out by its writer and cannot be checked again here.' }) : null]), + el('td', { cls: 'num', text: n(sumOver(s.id, moved)) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.up; })) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.directions.down; })) }), + el('td', { cls: 'num', text: verdict('as claimed') }), + el('td', { cls: 'num', text: verdict('off claim') }), + el('td', { cls: 'num', text: verdict('not claimed') }), + el('td', { cls: 'num', text: verdict('claimed, not moved') }), + el('td', { cls: 'num', text: verdict('within noise') }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.before + b.missing.both; })) }), + el('td', { cls: 'num', text: n(sumOver(s.id, function (b) { return b.missing.after + b.missing.both; })) }), + el('td', { cls: 'num', text: String(s.tolerance.value), title: s.tolerance.basis + (s.tolerance.note ? ': ' + s.tolerance.note : '') }), + el('td', { text: noise }) + ])); + }); + table.appendChild(body); - function renderWorst() { - var k = stepIndex(view.comp); - var rows = []; - D.items.forEach(function (item) { + var total = clear(q('total-table')); + q('total-heading').textContent = 'The whole way: ' + D.states[0].label + ' → ' + D.states[D.states.length - 1].label; + var classes = Object.keys(D.counts.total[D.indicators[0].id].classes); + var cols = ['Indicator'].concat(classes, ['Up', 'Down', 'Largest step', 'Spread over steps', 'Not attributable', 'Moved by what nothing explains']); + total.appendChild(el('thead', null, [el('tr', null, cols.map(function (h) { return el('th', { scope: 'col', text: h, title: SAID[h] || null }); }))])); + var tbody = el('tbody'); D.indicators.forEach(function (ind) { - if (view.worstIndicator && ind.id !== view.worstIndicator) return; - var cls = item.classes[ind.id][k]; - if (QUIET.indexOf(cls) >= 0) return; - if (!sizeAtLeast(cls, item.bands[ind.id][k], view.size)) return; - var pair = pairOf(item, ind.id, k); - rows.push({ item: item, ind: ind.id, key: severityKey(pair[0], pair[1], cls).concat([item.id, ind.id]) }); + var block = D.counts.total[ind.id]; + var att = D.attribution[ind.id]; + var ranked = Object.keys(att.dominant).filter(function (k) { return att.dominant[k]; }) + .sort(function (a, b) { return att.dominant[b] - att.dominant[a]; }) + .map(function (k) { return D.steps[stepIndex(k)].label + ' (' + att.dominant[k] + ')'; }); + var row = [el('th', { scope: 'row', text: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') })]; + classes.forEach(function (c) { + var title = c === 'missing' ? 'missing before: ' + block.missing.before + ', after: ' + block.missing.after + ', on both sides: ' + block.missing.both : null; + row.push(el('td', { cls: 'num' + (block.classes[c] ? '' : ' zero'), text: n(block.classes[c]), title: title })); + }); + row.push(el('td', { cls: 'num', text: n(block.directions.up) }), el('td', { cls: 'num', text: n(block.directions.down) })); + row.push(el('td', { text: ranked.join(', ') || 'none' })); + row.push(el('td', { cls: 'num', text: n(att.spread), title: 'its total moved, and no single step moved it: small steps that add up' })); + row.push(el('td', { cls: 'num', text: n(att.not_attributable), title: 'its total moved, and it has no result in a state between: its difference cannot be split' })); + row.push(el('td', { cls: 'num', text: n(att.unexplained_moved), title: 'moved in a step that no cause explains' })); + tbody.appendChild(el('tr', null, row)); }); - }); - rows.sort(function (a, b) { return compareKeys(a.key, b.key); }); - var shown = rows.slice(0, view.worstN); - var c = D.items_count; - q('worst-said').textContent = D.tier === 'counts' - ? 'This file holds counts only: it names no ' + D.item.one + '.' - : 'Showing ' + shown.length + ' of the ' + rows.length + ' values that moved or are missing in ' + compLabel(view.comp) + - ', among the ' + groupThousands(c.listed) + ' ' + D.item.many + ' in this file' + - (c.not_listed ? '. ' + groupThousands(c.not_listed) + ' more ' + D.item.many + ' worth a look are not in this file.' : '.') + - ' Ordered by class (sign change, zero reference, zero result, beyond 10x, differs, missing), then by how far the two sizes lie apart.' + sizeSaid(); - var table = clear(q('worst-table')); - var head = ['#', D.item.one.charAt(0).toUpperCase() + D.item.one.slice(1), 'Indicator', 'Class', 'Before', 'After', 'Change', 'Verdict']; - table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); - var body = el('tbody'); - shown.forEach(function (row, i) { - var pair = pairOf(row.item, row.ind, k); - var tr = el('tr', { cls: 'clickable', tabindex: '0', 'data-item': row.item.id }, [ - el('td', { cls: 'num', text: String(i + 1) }), - el('td', { text: row.item.name + (row.item.place ? ' (' + row.item.place + ')' : '') }), - el('td', { text: indicatorName(row.ind) }), - el('td', { text: row.item.classes[row.ind][k] }), - el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), - el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), - cell(row.item, row.ind, k), - el('td', { text: verdictOf(row.item, row.ind, k) || '—' }) - ]); - body.appendChild(tr); - }); - table.appendChild(body); - } + total.appendChild(tbody); + } - function gridRows() { - var k = stepIndex(view.comp); - var find = view.find.toLowerCase(); - return D.items.filter(function (item) { - if (find && (item.name + ' ' + (item.place || '') + ' ' + item.id).toLowerCase().indexOf(find) < 0) return false; - if (view.group && item.group !== view.group) return false; - var ids = D.indicators.map(function (i) { return i.id; }); - if (view.cls && !ids.some(function (id) { return item.classes[id][k] === view.cls; })) return false; - if (view.verdict && !ids.some(function (id) { return verdictOf(item, id, k) === view.verdict; })) return false; - if (view.size && !ids.some(function (id) { return QUIET.indexOf(item.classes[id][k]) < 0 && sizeAtLeast(item.classes[id][k], item.bands[id][k], view.size); })) return false; - return true; - }); - } + function renderDistributions() { + var comp = view.comp; + var blocks = comp === 'total' ? D.counts.total : D.counts.steps[comp]; + var first = blocks[D.indicators[0].id]; + var bands = Object.keys(first.bands); + var down = bands.filter(function (b) { return b.indexOf('down') === 0; }); + var up = bands.filter(function (b) { return b.indexOf('up') === 0; }); + var middle = ['agrees', 'within noise']; + var aside = ['zero reference', 'zero result', 'sign change', 'near zero', 'missing']; + var columns = down.map(function (b) { return ['band', b, 'down']; }) + .concat(middle.map(function (c) { return ['class', c, '']; })) + .concat(up.map(function (b) { return ['band', b, 'up']; })) + .concat(aside.map(function (c) { return ['class', c, '']; })); + var table = clear(q('dist-table')); + var head = [el('th', { scope: 'col', text: 'Indicator' })]; + columns.forEach(function (c) { + var label = c[0] === 'band' ? (c[2] === 'up' ? '↑ ' : '↓ ') + bandStep(c[1]) : (SHORT[c[1]] || c[1]); + head.push(el('th', { scope: 'col', cls: 'num', text: label, title: c[1] + (SAID[c[1]] ? ': ' + SAID[c[1]] : '') })); + }); + table.appendChild(el('thead', null, [el('tr', null, head)])); + var body = el('tbody'); + D.indicators.forEach(function (ind) { + var block = blocks[ind.id]; + var row = [el('th', { scope: 'row', text: indicatorName(ind.id) })]; + columns.forEach(function (c) { + var count = c[0] === 'band' ? block.bands[c[1]] : block.classes[c[1]]; + var share = block.pairs ? count / block.pairs : 0; + var bar = el('div', { cls: 'bar ' + c[2] }, [el('span', { style: 'height:' + (count ? Math.max(1, Math.round(share * 100)) : 0) + '%' })]); + row.push(el('td', { cls: 'bar-cell', title: c[1] + ': ' + count + ' of ' + block.pairs }, [bar, el('div', { cls: 'bar-n', text: groupThousands(count) })])); + }); + body.appendChild(el('tr', null, row)); + }); + table.appendChild(body); + } - function renderGrid() { - var k = stepIndex(view.comp); - var rows = gridRows(); - var shown = rows.slice(0, view.limit); - var c = D.items_count; - q('grid-said').textContent = D.tier === 'counts' - ? 'This file holds counts only: it names no ' + D.item.one + '.' - : 'Showing ' + shown.length + ' of ' + rows.length + ' ' + D.item.many + ' (' + groupThousands(c.listed) + ' in this file' + - (c.not_listed ? '; ' + groupThousands(c.not_listed) + ' more worth a look are not in it' : '') + '). ' + compLabel(view.comp) + '.' + sizeSaid(); - var table = clear(q('grid-table')); - var head = [el('th', { scope: 'col', text: D.item.one.charAt(0).toUpperCase() + D.item.one.slice(1) }), el('th', { scope: 'col', text: 'Unit' })]; - D.indicators.forEach(function (ind) { head.push(el('th', { scope: 'col', cls: 'num', text: indicatorName(ind.id), title: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') })); }); - table.appendChild(el('thead', null, [el('tr', null, head)])); - var body = el('tbody'); - shown.forEach(function (item) { - var row = [el('th', { scope: 'row', text: item.name + (item.place ? ' (' + item.place + ')' : '') }), el('td', { text: item.unit || '?' })]; - D.indicators.forEach(function (ind) { row.push(cell(item, ind.id, k)); }); - body.appendChild(el('tr', { cls: 'clickable', tabindex: '0', 'data-item': item.id }, row)); - }); - table.appendChild(body); - var more = q('grid-more'); - more.hidden = rows.length <= shown.length; - more.textContent = 'Show ' + Math.min(100, rows.length - shown.length) + ' more'; - } + function sizeSaid() { + return view.size ? ' Only changes by a factor of at least ' + view.size + ' are shown, with every zero, sign change and missing value: a filter of yours, which changes no count.' : ''; + } - function sizeSaid() { - return view.size ? ' Only changes by a factor of at least ' + view.size + ' are shown, with every zero, sign change and missing value: a filter of yours, which changes no count.' : ''; - } + function renderWorst() { + var k = stepIndex(view.comp); + var found = []; + rows().forEach(function (row) { + D.indicators.forEach(function (ind) { + if (view.worstIndicator && ind.id !== view.worstIndicator) return; + var cls = row.classes[ind.id][k]; + if (QUIET.indexOf(cls) >= 0) return; + if (!sizeAtLeast(cls, bandAt(row, ind.id, k), view.size)) return; + var pair = pairOf(row, ind.id, k); + found.push({ row: row, ind: ind.id, key: severityKey(pair[0], pair[1], cls).concat([row.id, ind.id]) }); + }); + }); + found.sort(function (a, b) { return compareKeys(a.key, b.key); }); + var shown = found.slice(0, view.worstN); + q('worst-said').textContent = D.tier === 'counts' + ? 'This file holds counts only: it names no ' + D.item.one + '.' + : 'Showing ' + shown.length + ' of the ' + groupThousands(found.length) + ' values that moved or are missing in ' + compLabel(view.comp) + + ', among ' + heldSaid() + '. Ordered by class (sign change, zero reference, zero result, beyond 10x, differs, missing), then by how far the two sizes lie apart.' + sizeSaid(); + var table = clear(q('worst-table')); + var head = ['#', capital(D.item.one), 'Indicator', 'Class', 'Before', 'After', 'Change', 'Verdict']; + table.appendChild(el('thead', null, [el('tr', null, head.map(function (h) { return el('th', { scope: 'col', text: h }); }))])); + var body = el('tbody'); + shown.forEach(function (one, i) { + var pair = pairOf(one.row, one.ind, k); + body.appendChild(el('tr', { cls: 'sd-row', tabindex: '0', 'data-item': one.row.id }, [ + el('td', { cls: 'num', text: String(i + 1) }), + el('td', { text: one.row.name + (one.row.place ? ' (' + one.row.place + ')' : '') }), + el('td', { text: indicatorName(one.ind) }), + el('td', { text: one.row.classes[one.ind][k] }), + el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), + el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), + valueCell(one.row, one.ind, k, false), + el('td', { text: verdictOf(one.row, one.ind, k) || '—' }) + ])); + }); + table.appendChild(body); + } - function fillSelect(select, options, chosen) { - clear(select); - options.forEach(function (o) { select.appendChild(el('option', { value: o[0], text: o[1] })); }); - select.value = chosen; - } + function gridRows() { + var k = stepIndex(view.comp); + var find = view.find.toLowerCase(); + var ids = indicatorIds(); + var found = rows().filter(function (row) { + if (find && (row.name + ' ' + (row.place || '') + ' ' + row.id).toLowerCase().indexOf(find) < 0) return false; + if (view.group && row.group !== view.group) return false; + if (view.cls && !ids.some(function (id) { return row.classes[id][k] === view.cls; })) return false; + if (view.verdict && !ids.some(function (id) { return verdictOf(row, id, k) === view.verdict; })) return false; + if (view.size && !ids.some(function (id) { return QUIET.indexOf(row.classes[id][k]) < 0 && sizeAtLeast(row.classes[id][k], bandAt(row, id, k), view.size); })) return false; + return true; + }); + if (!view.sort) return found; + var keyed = found.map(function (row, at) { + if (view.sort === 'name') return { row: row, key: [row.name.toLowerCase(), row.place || '', at] }; + var pair = pairOf(row, view.sort, k); + return { row: row, key: severityKey(pair[0], pair[1], row.classes[view.sort][k]).concat([at]) }; + }); + keyed.sort(function (a, b) { return compareKeys(a.key, b.key); }); + return keyed.map(function (one) { return one.row; }); + } - function renderFilters() { - var k = stepIndex(view.comp); - var classes = {}, verdicts = {}, groups = {}; - D.items.forEach(function (item) { + function sortButton(label, key, title) { + var button = el('button', { type: 'button', cls: 'sd-sort', text: label, title: title }); + button.addEventListener('click', function () { + view.sort = view.sort === key ? null : key; + view.limit = PAGE; + renderGrid(); + }); + return button; + } + + function renderGrid() { + var k = stepIndex(view.comp); + var found = gridRows(); + var shown = found.slice(0, view.limit); + q('grid-said').textContent = D.tier === 'counts' + ? 'This file holds counts only: it names no ' + D.item.one + '.' + : 'Showing ' + groupThousands(shown.length) + ' of ' + groupThousands(found.length) + ' ' + many(found.length) + ', among ' + heldSaid() + '. ' + + compLabel(view.comp) + '.' + sizeSaid(); + var table = clear(q('grid-table')); + var sorted = function (key) { return view.sort === key ? (key === 'name' ? 'ascending' : 'descending') : null; }; + var head = [ + el('th', { scope: 'col', cls: 'sd-name', 'aria-sort': sorted('name') }, [sortButton(capital(D.item.one), 'name', 'Sort by name')]), + el('th', { scope: 'col', cls: 'sd-group', text: 'Group' }), + el('th', { scope: 'col', text: 'Unit' }) + ]; D.indicators.forEach(function (ind) { - classes[item.classes[ind.id][k]] = 1; - var v = verdictOf(item, ind.id, k); - if (v) verdicts[v] = 1; + head.push(el('th', { scope: 'col', cls: 'num', 'aria-sort': sorted(ind.id), title: ind.name + (ind.unit ? ' (' + ind.unit + ')' : '') }, + [sortButton(indicatorName(ind.id), ind.id, 'Sort by how far the two lie apart, the furthest first')])); }); - if (item.group) groups[item.group] = 1; - }); - fillSelect(q('grid-class'), [['', 'Any']].concat(Object.keys(classes).sort().map(function (c) { return [c, c]; })), view.cls); - fillSelect(q('grid-verdict'), [['', 'Any']].concat(Object.keys(verdicts).sort().map(function (c) { return [c, c]; })), view.verdict); - fillSelect(q('grid-group'), [['', 'Any']].concat(Object.keys(groups).sort().map(function (c) { return [c, c]; })), view.group); - q('grid-verdict').disabled = !Object.keys(verdicts).length; - fillSelect(q('worst-indicator'), [['', 'All']].concat(D.indicators.map(function (i) { return [i.id, indicatorName(i.id)]; })), view.worstIndicator); - } + table.appendChild(el('thead', null, [el('tr', null, head)])); + var body = el('tbody'); + shown.forEach(function (row) { + var name = row.name + (row.place ? ' (' + row.place + ')' : ''); + var cells = [ + el('th', { scope: 'row', title: name }, [M && row.item ? el('span', { cls: 'sd-listed', title: 'listed in this file' }) : null, name]), + el('td', { cls: 'sd-group', text: row.group || '' }), + el('td', { text: row.unit || '?' }) + ]; + D.indicators.forEach(function (ind) { cells.push(valueCell(row, ind.id, k, true)); }); + body.appendChild(el('tr', { cls: 'sd-row', tabindex: '0', 'data-item': row.id, 'aria-current': view.open === row.id ? 'true' : null }, cells)); + }); + table.appendChild(body); + var more = q('grid-more'); + more.hidden = found.length <= shown.length; + more.textContent = 'Show ' + groupThousands(Math.min(PAGE, found.length - shown.length)) + ' more'; + } - function badge(by) { - if (!by) return el('span', { cls: 'badge', text: 'author not said' }); - if (by.kind === 'person') return el('span', { cls: 'badge person', text: 'Person: ' + by.name }); - if (by.kind === 'code') return el('span', { cls: 'badge', text: 'Code: ' + by.name }); - if (by.kind === 'model') return el('span', { cls: 'badge model', text: 'Model: ' + by.model + ', run by ' + by.run_by + '. A claim, not checked by a person' }); - return el('span', { cls: 'badge', text: String(by.kind) }); - } + function fillSelect(select, options, chosen) { + clear(select); + options.forEach(function (o) { select.appendChild(el('option', { value: o[0], text: o[1] })); }); + select.value = chosen; + } - function explanationNode(one) { - var about = []; - if (one.about.step) about.push(one.about.step === 'total' ? 'the whole way' : 'step ' + D.steps[stepIndex(one.about.step)].label); - if (one.about.indicator) about.push(indicatorName(one.about.indicator)); - if (one.about.item) { - var item = D.items.filter(function (i) { return i.id === one.about.item; })[0]; - about.push(item ? item.name : one.about.item); - } - return el('li', null, [ - badge(one.by), el('span', { cls: 'quiet', text: 'About ' + about.join(', ') + '. ' }), - one.finding === 'none found' ? el('strong', { text: 'Looked, found no reason: ' }) : null, one.text, - one.not_checked ? el('div', { cls: 'quiet', text: 'Not checked: ' + one.not_checked }) : null, - one.source ? el('div', { cls: 'quiet' }, [safeLink(one.source.link) ? el('a', { href: safeLink(one.source.link), text: one.source.link }) : null, one.source.doc ? ' ' + one.source.doc : null]) : null - ]); - } + function renderFilters() { + var k = stepIndex(view.comp); + var classes = {}, verdicts = {}, groups = {}; + var ids = indicatorIds(); + rows().forEach(function (row) { + ids.forEach(function (id) { + classes[row.classes[id][k]] = 1; + var v = verdictOf(row, id, k); + if (v) verdicts[v] = 1; + }); + if (row.group) groups[row.group] = 1; + }); + var any = [['', 'Any']]; + fillSelect(q('grid-class'), any.concat(Object.keys(classes).sort().map(function (c) { return [c, c]; })), view.cls); + fillSelect(q('grid-verdict'), any.concat(Object.keys(verdicts).sort().map(function (c) { return [c, c]; })), view.verdict); + fillSelect(q('grid-group'), any.concat(Object.keys(groups).sort().map(function (c) { return [c, c]; })), view.group); + q('grid-verdict').disabled = !Object.keys(verdicts).length; + fillSelect(q('worst-indicator'), [['', 'All']].concat(D.indicators.map(function (i) { return [i.id, indicatorName(i.id)]; })), view.worstIndicator); + } - function renderExplanations() { - var list = clear(q('explanation-list')); - if (!D.explanations.length) list.appendChild(el('li', { cls: 'quiet', text: 'The file holds no explanation.' })); - D.explanations.forEach(function (one) { list.appendChild(explanationNode(one)); }); - } + function badge(by) { + if (!by) return el('span', { cls: 'badge', text: 'author not said' }); + if (by.kind === 'person') return el('span', { cls: 'badge person', text: 'Person: ' + by.name }); + if (by.kind === 'code') return el('span', { cls: 'badge', text: 'Code: ' + by.name }); + if (by.kind === 'model') return el('span', { cls: 'badge model', text: 'Model: ' + by.model + ', run by ' + by.run_by + '. A claim, not checked by a person' }); + return el('span', { cls: 'badge', text: String(by.kind) }); + } - function renderRule() { - var body = clear(q('rule-body')); - body.appendChild(el('p', null, ['Every value is put into one class by one rule, in this order: missing, near zero, agrees, within noise (both only where the two share a sign), zero reference, zero result, sign change, beyond 10x, differs. ' + - 'Direction is up where the result after is larger as a signed number, down otherwise; colour follows the direction only. ' + - 'Beyond 10x: one size more than ' + D.rule.beyond + ' times the other. The whole way is counted with a tolerance of ' + D.rule.total_tolerance + ', the largest of the steps.'])); - var stepsTable = el('table', null, [el('thead', null, [el('tr', null, ['Step', 'Tolerance', 'Basis', 'Note', 'Noise'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); - var tb = el('tbody'); - D.steps.forEach(function (s) { - var noise = noiseText(s.noise, D.indicators.map(function (i) { return i.id; })) + (s.noise.measured ? ': ' + Object.keys(s.noise.by).map(function (k) { return indicatorName(k) + ' ' + formatNumber(s.noise.by[k]); }).join(', ') : ''); - tb.appendChild(el('tr', null, [el('td', { text: s.label }), el('td', { cls: 'num', text: String(s.tolerance.value) }), el('td', { text: s.tolerance.basis }), el('td', { text: s.tolerance.note || '' }), el('td', { text: noise })])); - }); - stepsTable.appendChild(tb); - body.appendChild(el('div', { cls: 'table-wrap' }, [stepsTable])); - body.appendChild(el('h3', { text: 'Floors' })); - body.appendChild(el('p', { cls: 'quiet', text: 'One floor per indicator and reference unit: ' + D.rule.floor_factor + ' times the median size of the results of ' + D.states[stepIndex2(D.rule.floors_from)].label + ' in that unit, unless declared. Units are never pooled.' })); - var floors = el('table', null, [el('thead', null, [el('tr', null, ['Indicator', 'Unit', 'Floor', 'Basis', 'From results'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); - var fb = el('tbody'); - D.rule.floors.forEach(function (f) { - fb.appendChild(el('tr', null, [el('td', { text: indicatorName(f.indicator) }), el('td', { text: f.unit || 'unknown' }), el('td', { cls: 'num', text: formatNumber(f.value), title: String(f.value) }), el('td', { text: f.basis }), el('td', { cls: 'num', text: n(f.n) })])); - }); - floors.appendChild(fb); - body.appendChild(el('div', { cls: 'table-wrap' }, [floors])); - var notes = clear(q('notes')); - D.notes.forEach(function (text) { notes.appendChild(el('li', { text: text })); }); - if (!D.notes.length) notes.appendChild(el('li', { cls: 'quiet', text: 'None.' })); - } - function stepIndex2(stateId) { - for (var i = 0; i < D.states.length; i++) if (D.states[i].id === stateId) return i; - return 0; - } + function aboutText(about) { + var parts = []; + if (about.step) parts.push(about.step === 'total' ? 'the whole way' : 'step ' + D.steps[stepIndex(about.step)].label); + if (about.indicator) parts.push(indicatorName(about.indicator)); + if (about.item) { + var row = rowById(about.item); + parts.push(row ? row.name : about.item); + } + return parts.length ? 'About ' + parts.join(', ') + '. ' : 'About the whole file. '; + } + + function explanationNode(one, scope) { + return el('li', null, [ + scope ? el('span', { cls: 'sd-scope', text: scope }) : null, + badge(one.by), el('span', { cls: 'sd-quiet', text: aboutText(one.about) }), + one.finding === 'none found' ? el('strong', { text: 'Looked, found no reason: ' }) : null, one.text, + one.not_checked ? el('div', { cls: 'sd-quiet', text: 'Not checked: ' + one.not_checked }) : null, + one.source ? el('div', { cls: 'sd-quiet' }, [safeLink(one.source.link) ? el('a', { href: safeLink(one.source.link), text: one.source.link }) : null, one.source.doc ? ' ' + one.source.doc : null]) : null + ]); + } - function renderItem(id) { - var item = D.items.filter(function (i) { return i.id === id; })[0]; - var section = q('item'); - if (!item) { section.hidden = true; return; } - section.hidden = false; - q('item-title').textContent = item.name; - q('item-close').onclick = function () { section.hidden = true; history.replaceState(null, '', location.pathname + location.search + '#grid'); }; - q('item-meta').textContent = [item.id, item.place, item.group, item.unit ? 'per ' + item.unit : 'unit unknown'].filter(Boolean).join(' · '); - var body = clear(q('item-body')); - D.indicators.forEach(function (ind) { - var values = item.values[ind.id]; + function renderExplanations() { + var list = clear(q('explanation-list')); + if (!D.explanations.length) list.appendChild(el('li', { cls: 'sd-quiet', text: 'The file holds no explanation.' })); + D.explanations.forEach(function (one) { list.appendChild(explanationNode(one)); }); + } + + function renderRule() { + var body = clear(q('rule-body')); + body.appendChild(el('p', null, ['Every value is put into one class by one rule, in this order: missing, near zero, agrees, within noise (both only where the two share a sign), zero reference, zero result, sign change, beyond 10x, differs. ' + + 'Direction is up where the result after is larger as a signed number, down otherwise; colour follows the direction only. ' + + 'Beyond 10x: one size more than ' + D.rule.beyond + ' times the other. The whole way is counted with a tolerance of ' + D.rule.total_tolerance + ', the largest of the steps. ' + + 'The matrix shows the result after divided by the result before; the ratio and the strength of its colour are a display, and change no class.'])); + var stepsTable = el('table', null, [el('thead', null, [el('tr', null, ['Step', 'Tolerance', 'Basis', 'Note', 'Noise'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var tb = el('tbody'); + D.steps.forEach(function (s) { + var noise = noiseText(s.noise, indicatorIds()) + (s.noise.measured ? ': ' + Object.keys(s.noise.by).map(function (k) { return indicatorName(k) + ' ' + formatNumber(s.noise.by[k]); }).join(', ') : ''); + tb.appendChild(el('tr', null, [el('td', { text: s.label }), el('td', { cls: 'num', text: String(s.tolerance.value) }), el('td', { text: s.tolerance.basis }), el('td', { text: s.tolerance.note || '' }), el('td', { text: noise })])); + }); + stepsTable.appendChild(tb); + body.appendChild(el('div', { cls: 'sd-wrap' }, [stepsTable])); + body.appendChild(el('h4', { text: 'Floors' })); + body.appendChild(el('p', { cls: 'sd-quiet', text: 'One floor per indicator and reference unit: ' + D.rule.floor_factor + ' times the median size of the results of ' + D.states[stateIndex(D.rule.floors_from)].label + ' in that unit, unless declared. Units are never pooled.' })); + var floors = el('table', null, [el('thead', null, [el('tr', null, ['Indicator', 'Unit', 'Floor', 'Basis', 'From results'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var fb = el('tbody'); + D.rule.floors.forEach(function (f) { + fb.appendChild(el('tr', null, [el('td', { text: indicatorName(f.indicator) }), el('td', { text: f.unit || 'unknown' }), el('td', { cls: 'num', text: formatNumber(f.value), title: String(f.value) }), el('td', { text: f.basis }), el('td', { cls: 'num', text: n(f.n) })])); + }); + floors.appendChild(fb); + body.appendChild(el('div', { cls: 'sd-wrap' }, [floors])); + var notes = clear(q('notes')); + D.notes.forEach(function (text) { notes.appendChild(el('li', { text: text })); }); + if (!D.notes.length) notes.appendChild(el('li', { cls: 'sd-quiet', text: 'None.' })); + } + + // ---------------------------------------------------------------- the item drawer + + // whether a row moved in a comparison and indicator an explanation is about; an + // explanation that names neither is about every one + function movedUnder(row, about) { + var ids = about.indicator ? [about.indicator] : indicatorIds(); + var ks = about.step ? [stepIndex(about.step)] : D.steps.map(function (s, k) { return k; }).concat([D.steps.length]); + return ids.some(function (id) { + return row.classes[id] && ks.some(function (k) { return MOVES.indexOf(row.classes[id][k]) >= 0; }); + }); + } + + function whyNode(row) { + var list = el('ul', { cls: 'sd-why' }); + var own = D.explanations.filter(function (e) { return e.about.item === row.id; }); + var wider = D.explanations.filter(function (e) { return !e.about.item && movedUnder(row, e.about); }); + own.forEach(function (e) { list.appendChild(explanationNode(e, 'This ' + D.item.one)); }); + wider.forEach(function (e) { list.appendChild(explanationNode(e, e.about.step && e.about.indicator ? 'Its step and indicator' : e.about.step ? 'A step it moved in' : e.about.indicator ? 'An indicator it moved in' : 'The whole file')); }); + if (!own.length && !wider.length) { + var any = movedUnder(row, {}); + list.appendChild(el('li', { cls: 'sd-none', text: any + ? 'None given: no explanation in this file is about this ' + D.item.one + ', or about a step or indicator in which it moved.' + : 'It moved in no comparison, and no explanation in this file is about it.' })); + } + return list; + } + + function itemTable(row, ind) { + var values = row.values[ind.id]; var first = values[0], last = values[values.length - 1]; - var totalCls = item.classes[ind.id][D.steps.length]; - body.appendChild(el('h3', { text: ind.name + (ind.unit ? ' (' + ind.unit + ' per ' + (item.unit || 'unit') + ')' : '') })); - var table = el('table', null, [el('thead', null, [el('tr', null, ['Step', 'Before', 'After', 'Change', 'Verdict', 'Share of the whole difference'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var totalCls = row.classes[ind.id][D.steps.length]; + var table = el('table', { 'data-indicator': ind.id }, [el('thead', null, [el('tr', null, ['Step', 'Before', 'After', 'Change', 'Verdict', 'Share of the whole difference'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); var tb = el('tbody'); var split = splitOf(values, D.steps.map(function (s) { return s.explained; }), totalCls); D.steps.forEach(function (s, k) { var b = values[k], a = values[k + 1]; var share = split.state === 'split' ? shareText(split.shares[k]) : '—'; tb.appendChild(el('tr', null, [ - el('td', null, ['Step ' + k + ': ' + s.label, s.explained ? null : el('div', { cls: 'error', text: 'nothing explains it' })]), + el('td', null, ['Step ' + k + ': ' + s.label, s.explained ? null : el('div', { cls: 'sd-error', text: 'nothing explains it' })]), el('td', { cls: 'num', text: formatNumber(b), title: String(b) }), el('td', { cls: 'num', text: formatNumber(a), title: String(a) }), - cell(item, ind.id, k), - el('td', { text: verdictOf(item, ind.id, k) || '—' }), + valueCell(row, ind.id, k, false), + el('td', { text: verdictOf(row, ind.id, k) || '—' }), el('td', { cls: 'num', text: share }) ])); }); @@ -670,182 +892,342 @@ el('th', { scope: 'row', text: 'The whole way' }), el('td', { cls: 'num', text: formatNumber(first), title: String(first) }), el('td', { cls: 'num', text: formatNumber(last), title: String(last) }), - cell(item, ind.id, D.steps.length), + valueCell(row, ind.id, D.steps.length, false), el('td', { text: '—' }), el('td', { cls: 'num', text: splitText(split) }) ])); table.appendChild(tb); - body.appendChild(el('div', { cls: 'table-wrap' }, [table])); - }); - var mine = D.explanations.filter(function (e) { return e.about.item === item.id; }); - if (mine.length) { - body.appendChild(el('h3', { text: 'Explanations' })); - body.appendChild(el('ul', { cls: 'plain' }, mine.map(explanationNode))); - } - Object.keys(item.breakdowns || {}).forEach(function (comp) { - var name = item.breakdowns[comp]; - var holder = el('div'); - body.appendChild(el('h3', { text: 'What it is made of: ' + compLabel(comp) })); - body.appendChild(holder); - var found = B[name]; - if (found) { - if (breakdownFits(found, item.id, comp)) breakdownInto(holder, found); - else holder.appendChild(el('p', { cls: 'error', text: 'The breakdown ' + name + ' is of another ' + D.item.one + ' or another step: not drawn.' })); - return; + return el('div', { cls: 'sd-wrap' }, [table]); + } + + function renderItem(id) { + var row = rowById(id); + var drawer = q('item'); + if (!row) { closeItem(); return; } + if (drawer.hidden) lastFocus = doc.activeElement || null; + view.open = row.id; + drawer.hidden = false; + q('item-title').textContent = row.name; + q('item-meta').textContent = [row.id, row.place, row.group, row.unit ? 'per ' + row.unit : 'unit unknown', + row.item ? null : 'from the matrix: not listed in this file'].filter(Boolean).join(' · '); + var body = clear(q('item-body')); + body.appendChild(el('h4', { text: 'Why it differs' })); + body.appendChild(whyNode(row)); + body.appendChild(el('p', { cls: 'sd-quiet', text: 'The explanations about this ' + D.item.one + ' first, then those about a step or indicator in which it moved: those may not be about it. An explanation is a claim; it changes no class.' })); + // the indicators, the one whose whole way lies furthest apart first + var last = D.steps.length; + var order = D.indicators.slice().sort(function (a, b) { + var pa = pairOf(row, a.id, last), pb = pairOf(row, b.id, last); + return compareKeys(severityKey(pa[0], pa[1], row.classes[a.id][last]), severityKey(pb[0], pb[1], row.classes[b.id][last])); + }); + order.forEach(function (ind) { + body.appendChild(el('h4', { text: ind.name + (ind.unit ? ' (' + ind.unit + ' per ' + (row.unit || 'unit') + ')' : '') })); + body.appendChild(itemTable(row, ind)); + }); + var named = row.item ? row.item.breakdowns || {} : {}; + Object.keys(named).forEach(function (comp) { + var name = named[comp]; + var holder = el('div'); + body.appendChild(el('h4', { text: 'What it is made of: ' + compLabel(comp) })); + body.appendChild(holder); + var found = B[name]; + if (found) { + if (breakdownFits(found, row.id, comp)) breakdownInto(holder, found); + else holder.appendChild(el('p', { cls: 'sd-error', text: 'The breakdown ' + name + ' is of another ' + D.item.one + ' or another step: not drawn.' })); + return; + } + holder.appendChild(el('p', { cls: 'sd-quiet', text: 'The breakdown ' + name + ' is not with this page.' })); + if (safeName(name) && /^https?:$/.test(win.location.protocol)) { + win.fetch(base + name).then(function (r) { if (!r.ok) throw new Error(); return r.json(); }) + .then(function (data) { if (!breakdownFits(data, row.id, comp)) throw new Error(); clear(holder); breakdownInto(holder, data); }) + .catch(function () { /* the sentence above stays */ }); + } + }); + markOpen(); + var title = q('item-title'); + if (title.focus) title.focus(); + } + + function markOpen() { + var table = q('grid-table'); + var trs = table.querySelectorAll ? table.querySelectorAll('tr[data-item]') : []; + for (var i = 0; i < trs.length; i++) { + if (trs[i].getAttribute('data-item') === view.open) trs[i].setAttribute('aria-current', 'true'); + else if (trs[i].removeAttribute) trs[i].removeAttribute('aria-current'); } - holder.appendChild(el('p', { cls: 'quiet', text: 'The breakdown ' + name + ' is not with this page.' })); - if (location.protocol === 'http:' || location.protocol === 'https:') { - fetch(name).then(function (r) { if (!r.ok) throw new Error(); return r.json(); }) - .then(function (data) { if (!breakdownFits(data, item.id, comp)) throw new Error(); clear(holder); breakdownInto(holder, data); }) - .catch(function () { /* the sentence above stays */ }); + } + + function closeItem() { + var drawer = q('item'); + if (!drawer || drawer.hidden) return; + drawer.hidden = true; + view.open = null; + markOpen(); + if (/^#item=/.test(win.location.hash || '') && win.history && win.history.replaceState) { + win.history.replaceState(null, '', win.location.pathname + win.location.search); } - }); - section.scrollIntoView({ block: 'start' }); - } + if (lastFocus && lastFocus.focus && root.contains && root.contains(lastFocus)) lastFocus.focus(); + lastFocus = null; + } - // a breakdown is drawn only under the item and the step that it breaks down - function breakdownFits(data, itemId, comp) { - if (!data || data.format !== 'sentier-diagnostics-breakdown/1' || data.item !== itemId) return false; - var k = stepIndex(comp); - var from = comp === 'total' ? D.states[0].id : D.steps[k].from; - var to = comp === 'total' ? D.states[D.states.length - 1].id : D.steps[k].to; - return data.from === from && data.to === to && !!data.indicators && typeof data.indicators === 'object'; - } + // a breakdown is drawn only under the item and the step that it breaks down + function breakdownFits(data, itemId, comp) { + if (!data || data.format !== 'sentier-diagnostics-breakdown/1' || data.item !== itemId) return false; + var k = stepIndex(comp); + var from = comp === 'total' ? D.states[0].id : D.steps[k].from; + var to = comp === 'total' ? D.states[D.states.length - 1].id : D.steps[k].to; + return data.from === from && data.to === to && !!data.indicators && typeof data.indicators === 'object'; + } - function breakdownInto(holder, data) { - Object.keys(data.indicators).forEach(function (key) { - var entry = data.indicators[key]; - holder.appendChild(el('h4', { text: indicatorName(key) + (entry.unit ? ' (' + entry.unit + ')' : '') })); - var table = el('table', null, [el('thead', null, [el('tr', null, ['Part', 'Kind', 'Before', 'After', 'Change'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); - var tb = el('tbody'); - var line = function (label, kind, pair, strong) { - var d = pair[1] - pair[0]; - return el('tr', null, [el(strong ? 'th' : 'td', { text: label }), el('td', { text: kind || '' }), - el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), - el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), - el('td', { cls: 'num cell ' + (d > 0 ? 'tone-up' : d < 0 ? 'tone-down' : 'tone-none'), style: '--s:0.4', text: d === 0 ? '=' : (d > 0 ? '+' : '−') + formatNumber(Math.abs(d)) })]); - }; - entry.parts.forEach(function (p) { tb.appendChild(line(p.label, p.kind, p.values, false)); }); - tb.appendChild(line(entry.parts_not_listed ? 'The other ' + entry.parts_not_listed + ' parts' : 'No other part', '', entry.rest, false)); - tb.appendChild(line('Total', '', entry.total, true)); - table.appendChild(tb); - holder.appendChild(el('div', { cls: 'table-wrap' }, [table])); - var cover = function (side) { - var listed = entry.parts.reduce(function (a, p) { return a + p.values[side]; }, 0); - return entry.total[side] ? shareText(listed / entry.total[side]) : 'none (the total is 0)'; - }; - holder.appendChild(el('p', { cls: 'quiet', text: (entry.parts_not_listed - ? 'The first ' + entry.parts.length + ' parts make ' + cover(0) + ' of the total before and ' + cover(1) + ' after; the other ' + entry.parts_not_listed + ' are summed in one line. ' - : 'Every part is listed. ') + - 'Parts are listed by the larger of their two sizes, so that a part large on one side only is listed. The listed parts and the others add up to the total.' })); - }); - } + function breakdownInto(holder, data) { + Object.keys(data.indicators).forEach(function (key) { + var entry = data.indicators[key]; + holder.appendChild(el('h4', { text: indicatorName(key) + (entry.unit ? ' (' + entry.unit + ')' : '') })); + var table = el('table', null, [el('thead', null, [el('tr', null, ['Part', 'Kind', 'Before', 'After', 'Change'].map(function (h) { return el('th', { scope: 'col', text: h }); }))])]); + var tb = el('tbody'); + var line = function (label, kind, pair, strong) { + var d = pair[1] - pair[0]; + return el('tr', null, [el(strong ? 'th' : 'td', { text: label }), el('td', { text: kind || '' }), + el('td', { cls: 'num', text: formatNumber(pair[0]), title: String(pair[0]) }), + el('td', { cls: 'num', text: formatNumber(pair[1]), title: String(pair[1]) }), + el('td', { cls: 'num cell ' + (d > 0 ? 'tone-up' : d < 0 ? 'tone-down' : 'tone-none'), style: '--s:0.4', text: d === 0 ? '=' : (d > 0 ? '+' : '−') + formatNumber(Math.abs(d)) })]); + }; + entry.parts.forEach(function (p) { tb.appendChild(line(p.label, p.kind, p.values, false)); }); + tb.appendChild(line(entry.parts_not_listed ? 'The other ' + entry.parts_not_listed + ' parts' : 'No other part', '', entry.rest, false)); + tb.appendChild(line('Total', '', entry.total, true)); + table.appendChild(tb); + holder.appendChild(el('div', { cls: 'sd-wrap' }, [table])); + var cover = function (side) { + var listed = entry.parts.reduce(function (a, p) { return a + p.values[side]; }, 0); + return entry.total[side] ? shareText(listed / entry.total[side]) : 'none (the total is 0)'; + }; + holder.appendChild(el('p', { cls: 'sd-quiet', text: (entry.parts_not_listed + ? 'The first ' + entry.parts.length + ' parts make ' + cover(0) + ' of the total before and ' + cover(1) + ' after; the other ' + entry.parts_not_listed + ' are summed in one line. ' + : 'Every part is listed. ') + + 'Parts are listed by the larger of their two sizes, so that a part large on one side only is listed. The listed parts and the others add up to the total.' })); + }); + } - function renderComparison() { - renderFilters(); - renderDistributions(); - renderWorst(); - renderGrid(); - } + // ---------------------------------------------------------------- the matrix - // a file the page cannot draw draws nothing: not even the part it drew before it failed - function refuse(problem) { - D = null; - document.title = 'Diagnostics'; - ['title', 'about', 'tier', 'audience', 'unchecked'].forEach(function (id) { - var node = q(id); - if (node) node.textContent = ''; - }); - ['audience', 'unchecked'].forEach(function (id) { if (q(id)) q(id).hidden = true; }); - if (q('states')) clear(q('states')); - q('load').hidden = false; - q('content').hidden = true; - q('load-error').textContent = problem; - } + function matrixSaid() { + var node = q('matrix-said'); + if (!node) return; + var c = D.items_count; + var text = { + none: D.tier === 'counts' ? '' : 'This file lists ' + groupThousands(c.listed) + ' of its ' + groupThousands(c.all) + ' ' + D.item.many + ' and names no matrix of every ' + D.item.one + ': the grid shows the listed ones.', + loading: 'Loading the matrix of every ' + D.item.one + ' (' + matrix.said + '); the grid shows the listed ones until it is here.', + loaded: 'The matrix of all ' + groupThousands(M ? M.length : 0) + ' ' + D.item.many + ' is loaded; a dot marks the ' + groupThousands(c.listed) + ' listed in this file.', + refused: 'The matrix was not loaded: ' + matrix.said + '. The grid shows the ' + groupThousands(c.listed) + ' ' + D.item.many + ' listed in this file.' + }[matrix.state]; + node.textContent = text; + } - // `checked`: the file came as data.js, which `sentier-diagnostics page` writes only after - // the validator passed it. A file chosen here or named in the address was not checked: the - // page checks its shape only, and says so above everything it draws. - function show(data, checked) { - var problem = checkShape(data); - if (problem) { refuse(problem); return; } - D = data; - view.comp = 'total'; - q('load').hidden = true; - q('content').hidden = false; - var unchecked = q('unchecked'); // a site's own frame may not have it; its data.js was checked - if (unchecked) { - unchecked.hidden = !!checked; - unchecked.textContent = checked ? '' : 'Opened in the browser: this page checked the shape of the file, not its counts. ' + - 'Run sentier-diagnostics check on it before relying on what it shows.'; - } - try { - renderHeader(); - renderSteps(); - renderExplanations(); - renderRule(); - legend(q('grid-legend')); - legend(q('dist-legend')); + function takeMatrix(m) { + if (!D) return; + var problem = checkMatrix(m, D); + if (problem) { matrix.state = 'refused'; matrix.said = problem; matrixSaid(); return; } + M = matrixRows(m, D); + matrix.state = 'loaded'; + matrix.said = ''; + view.limit = PAGE; renderComparison(); - route(); - } catch (error) { - refuse('The file is JSON of the right format, but a part of it is missing or not what the format says (' + error.message + '). Run sentier-diagnostics check on it.'); + matrixSaid(); + if (view.open) renderItem(view.open); + else route(); } - } - function route() { - var hash = decodeURIComponent(location.hash.slice(1)); - if (hash.indexOf('item=') === 0) renderItem(hash.slice(5)); - } + function refuseMatrix(said) { matrix.state = 'refused'; matrix.said = said; matrixSaid(); } - function openFrom(event) { - var row = event.target.closest ? event.target.closest('tr[data-item]') : null; - if (row) location.hash = 'item=' + encodeURIComponent(row.getAttribute('data-item')); - } + // the matrix comes after the page is drawn: from a global, or from the file the root or + // the diagnostics file names, as a script beside the page (which opens from disk too) or + // as JSON over http + function loadMatrix() { + if (given.matrix) { takeMatrix(given.matrix); return; } + if (win.SENTIER_DIAGNOSTICS_MATRIX) { takeMatrix(win.SENTIER_DIAGNOSTICS_MATRIX); return; } + var name = root.getAttribute('data-sd-matrix') || (D.matrix ? D.matrix.file : null); + if (!name || D.tier === 'counts') { matrixSaid(); return; } + if (!safeName(name)) { refuseMatrix('its name is not a plain file name beside the page'); return; } + matrix.state = 'loading'; + matrix.said = name; + matrixSaid(); + if (/\.js$/.test(name)) { + var script = doc.createElement('script'); + script.setAttribute('src', base + name); + script.onload = function () { + if (win.SENTIER_DIAGNOSTICS_MATRIX) takeMatrix(win.SENTIER_DIAGNOSTICS_MATRIX); + else refuseMatrix(name + ' holds no matrix'); + }; + script.onerror = function () { refuseMatrix(name + ' is not beside the page'); }; + (doc.head || doc.body || root).appendChild(script); + return; + } + if (!/^https?:$/.test(win.location.protocol)) { + refuseMatrix(name + ' is read over http only; `sentier-diagnostics page` writes it beside the page as a script, which opens from disk'); + return; + } + win.fetch(base + name).then(function (r) { + if (!r.ok) throw new Error(name + ' could not be read (' + r.status + ')'); + return r.text(); + }).then(function (text) { + if (text.length > MATRIX_MAX_BYTES) throw new Error(name + ' is larger than ' + groupThousands(MATRIX_MAX_BYTES) + ' bytes'); + var m; + try { m = JSON.parse(text); } catch (error) { throw new Error(name + ' is not JSON'); } + takeMatrix(m); + }).catch(function (error) { refuseMatrix(error.message || (name + ' could not be read')); }); + } - function start() { - B = root.SENTIER_DIAGNOSTICS_BREAKDOWNS || {}; - q('comparison').addEventListener('change', function (e) { view.comp = e.target.value; view.cls = ''; view.verdict = ''; view.limit = 100; renderComparison(); }); - q('size').addEventListener('change', function (e) { view.size = +e.target.value; view.limit = 100; renderWorst(); renderGrid(); }); - q('worst-n').addEventListener('change', function (e) { view.worstN = +e.target.value; renderWorst(); }); - q('worst-indicator').addEventListener('change', function (e) { view.worstIndicator = e.target.value; renderWorst(); }); - q('grid-find').addEventListener('input', function (e) { view.find = e.target.value; view.limit = 100; renderGrid(); }); - q('grid-class').addEventListener('change', function (e) { view.cls = e.target.value; view.limit = 100; renderGrid(); }); - q('grid-verdict').addEventListener('change', function (e) { view.verdict = e.target.value; view.limit = 100; renderGrid(); }); - q('grid-group').addEventListener('change', function (e) { view.group = e.target.value; view.limit = 100; renderGrid(); }); - q('grid-more').addEventListener('click', function () { view.limit += 100; renderGrid(); }); - ['grid-table', 'worst-table'].forEach(function (id) { - q(id).addEventListener('click', openFrom); - q(id).addEventListener('keydown', function (e) { if (e.key === 'Enter') openFrom(e); }); - }); - window.addEventListener('hashchange', route); - q('file').addEventListener('change', function (e) { - var file = e.target.files[0]; - if (!file) return; - var reader = new FileReader(); - reader.onload = function () { - var data; - try { data = JSON.parse(reader.result); } catch (error) { refuse('The file is not JSON: ' + error.message); return; } - show(data, false); - }; - reader.readAsText(file); - }); - if (root.SENTIER_DIAGNOSTICS) { show(root.SENTIER_DIAGNOSTICS, true); return; } - var wanted = new URLSearchParams(location.search).get('file'); - if (wanted && /^[A-Za-z0-9][A-Za-z0-9._\/-]*\.json$/.test(wanted) && wanted.indexOf('..') < 0 && location.protocol.indexOf('http') === 0) { - fetch(wanted).then(function (r) { return r.json(); }).then(function (data) { show(data, false); }, function () { - refuse('Could not read ' + wanted + ' as JSON.'); + // ---------------------------------------------------------------- drawing a file + + function renderComparison() { + renderFilters(); + renderDistributions(); + renderWorst(); + renderGrid(); + } + + // a file the page cannot draw draws nothing: not even the part it drew before it failed + function refuse(problem) { + D = null; M = null; LISTED = []; + if (standalone) doc.title = 'Diagnostics'; + ['title', 'about', 'tier', 'audience', 'unchecked', 'matrix-said'].forEach(function (name) { + var node = q(name); + if (node) node.textContent = ''; + }); + ['audience', 'unchecked', 'item'].forEach(function (name) { if (q(name)) q(name).hidden = true; }); + if (q('states')) clear(q('states')); + q('load').hidden = false; + q('content').hidden = true; + q('load-error').textContent = problem; + } + + // `checked`: the file came with the page, which `sentier-diagnostics page` writes only + // after the validator passed it. A file chosen here or named in the address was not + // checked: the page checks its shape only, and says so above everything it draws. + function show(data, checked) { + var problem = checkShape(data); + if (problem) { refuse(problem); return; } + D = data; + M = null; + LISTED = D.items.map(listedRow); + matrix = { state: 'none', said: '' }; + view.comp = 'total'; + view.limit = PAGE; + q('load').hidden = true; + q('content').hidden = false; + var unchecked = q('unchecked'); + if (unchecked) { + unchecked.hidden = !!checked; + unchecked.textContent = checked ? '' : 'Opened in the browser: this page checked the shape of the file, not its counts. ' + + 'Run sentier-diagnostics check on it before relying on what it shows.'; + } + try { + renderHeader(); + renderSteps(); + renderExplanations(); + renderRule(); + legend(q('grid-legend'), true); + legend(q('dist-legend'), false); + renderComparison(); + matrixSaid(); + route(); + } catch (error) { + refuse('The file is JSON of the right format, but a part of it is missing or not what the format says (' + error.message + '). Run sentier-diagnostics check on it.'); + return; + } + if (checked) loadMatrix(); + } + + function route() { + if (!D) return; + var hash = decodeURIComponent((win.location.hash || '').slice(1)); + if (hash.indexOf('item=') === 0) renderItem(hash.slice(5)); + else closeItem(); + } + + function openFrom(event) { + var row = event.target.closest ? event.target.closest('tr[data-item]') : null; + if (!row) return; + var id = row.getAttribute('data-item'); + if (win.location.hash === '#item=' + encodeURIComponent(id)) renderItem(id); + else win.location.hash = 'item=' + encodeURIComponent(id); + } + + function readScript(name) { + var node = q(name); + if (!node || !node.textContent || !/\S/.test(node.textContent)) return undefined; + return JSON.parse(node.textContent); + } + + function start() { + q('comparison').addEventListener('change', function (e) { view.comp = e.target.value; view.cls = ''; view.verdict = ''; view.limit = PAGE; renderComparison(); }); + q('size').addEventListener('change', function (e) { view.size = +e.target.value; view.limit = PAGE; renderWorst(); renderGrid(); }); + q('worst-n').addEventListener('change', function (e) { view.worstN = +e.target.value; renderWorst(); }); + q('worst-indicator').addEventListener('change', function (e) { view.worstIndicator = e.target.value; renderWorst(); }); + q('grid-find').addEventListener('input', function (e) { view.find = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-class').addEventListener('change', function (e) { view.cls = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-verdict').addEventListener('change', function (e) { view.verdict = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-group').addEventListener('change', function (e) { view.group = e.target.value; view.limit = PAGE; renderGrid(); }); + q('grid-more').addEventListener('click', function () { view.limit += PAGE; renderGrid(); }); + q('item-close').addEventListener('click', closeItem); + root.addEventListener('keydown', function (e) { if (e.key === 'Escape') closeItem(); }); + ['grid-table', 'worst-table'].forEach(function (name) { + q(name).addEventListener('click', openFrom); + q(name).addEventListener('keydown', function (e) { if (e.key === 'Enter') openFrom(e); }); }); - return; + win.addEventListener('hashchange', route); + q('file').addEventListener('change', function (e) { + var file = e.target.files[0]; + if (!file) return; + var reader = new win.FileReader(); + reader.onload = function () { + var data; + try { data = JSON.parse(reader.result); } catch (error) { refuse('The file is not JSON: ' + error.message); return; } + show(data, false); + }; + reader.readAsText(file); + }); + try { + var breakdowns = given.breakdowns || readScript('breakdowns'); + B = breakdowns || win.SENTIER_DIAGNOSTICS_BREAKDOWNS || {}; + var data = given.data || readScript('data'); + if (data === undefined) data = win.SENTIER_DIAGNOSTICS; + } catch (error) { + refuse('The data of this page is not JSON: ' + error.message); + return; + } + if (data) { show(data, true); return; } + var wanted = new win.URLSearchParams(win.location.search).get('file'); + if (wanted && safeName(wanted) && /\.json$/.test(wanted) && /^https?:$/.test(win.location.protocol)) { + win.fetch(base + wanted).then(function (r) { return r.json(); }).then(function (got) { show(got, false); }, function () { + refuse('Could not read ' + wanted + ' as JSON.'); + }); + return; + } + q('load').hidden = false; + } + + start(); + return { show: show, close: closeItem, open: renderItem }; + } + + // every root of the document not yet mounted, but those a host mounts itself + function mountAll(doc) { + var roots = (doc || global.document).querySelectorAll('[data-sentier-diagnostics]'); + for (var i = 0; i < roots.length; i++) { + if (roots[i].getAttribute('data-sentier-diagnostics') !== 'manual') mount(roots[i]); } - q('load').hidden = false; } - var api = { formatNumber: formatNumber, percentText: percentText, shareText: shareText, noiseText: noiseText, sizeAtLeast: sizeAtLeast, cellView: cellView, severityKey: severityKey, - compareKeys: compareKeys, bandStep: bandStep, checkShape: checkShape, safeLink: safeLink, splitOf: splitOf, splitText: splitText, distance: distance, RANK: RANK }; + var api = { formatNumber: formatNumber, percentText: percentText, shareText: shareText, ratioText: ratioText, noiseText: noiseText, + sizeAtLeast: sizeAtLeast, cellView: cellView, ratioView: ratioView, severityKey: severityKey, compareKeys: compareKeys, + bandStep: bandStep, bandOf: bandOf, checkShape: checkShape, checkMatrix: checkMatrix, matrixRows: matrixRows, + safeLink: safeLink, safeName: safeName, splitOf: splitOf, splitText: splitText, distance: distance, RANK: RANK, + mount: mount, mountAll: mountAll }; if (typeof module !== 'undefined' && module.exports) module.exports = api; - else if (typeof document !== 'undefined') { - if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', start); - else start(); + else if (typeof global.document !== 'undefined') { + global.SentierDiagnostics = api; + if (global.document.readyState === 'loading') global.document.addEventListener('DOMContentLoaded', function () { mountAll(); }); + else mountAll(); } })(typeof window !== 'undefined' ? window : this); diff --git a/src/sentier_diagnostics/page/index.html b/src/sentier_diagnostics/page/index.html index a76214a..50960be 100644 --- a/src/sentier_diagnostics/page/index.html +++ b/src/sentier_diagnostics/page/index.html @@ -3,118 +3,14 @@ -Diagnostics +<!--title--> + - - - -
    -

    sentier-diagnostics

    -

    Diagnostics

    -

    -
      - - -

      -
      - - - -
      - - - +
      +
      - -

      diff --git a/src/sentier_diagnostics/page/standalone.css b/src/sentier_diagnostics/page/standalone.css new file mode 100644 index 0000000..75e0671 --- /dev/null +++ b/src/sentier_diagnostics/page/standalone.css @@ -0,0 +1,37 @@ +/* The frame of a page that stands on its own (`sentier-diagnostics page`). A site that holds + the diagnostics in its own page loads diagnostics.css only: this sheet sets the tokens a + site would set, the font and the ground, which diagnostics.css leaves to its host. */ +:root { + --ink: #1d232a; + --muted: #5b6672; + --surface: #ffffff; + --paper-deep: #f3f5f7; + --line: #d9dee3; + --line-strong: #b6bfc8; + --index: #1d625e; + --fail: #a3262a; + --radius: 6px; + --gutter: 16px; + color-scheme: light dark; +} +@media (prefers-color-scheme: dark) { + :root { + --ink: #e6eaee; + --muted: #9aa5b0; + --surface: #15191d; + --paper-deep: #1f252b; + --line: #36404a; + --line-strong: #56616c; + --index: #739e9b; + --fail: #ff8a8e; + } +} +body { + margin: 0; + font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + line-height: 1.45; + color: var(--ink); + background: var(--surface); +} +.sd-standalone { max-width: 1400px; margin: 0 auto; padding: 0 var(--gutter) 2rem; } +.sd-standalone h1 { font-size: 1.6rem; margin: 1rem 0 0.2rem; } diff --git a/tests/js/dom.js b/tests/js/dom.js index d4c81e1..9bdbcec 100644 --- a/tests/js/dom.js +++ b/tests/js/dom.js @@ -1,7 +1,7 @@ // A small stand-in for the part of the DOM the page script uses, so that the page can be // drawn under Node without a browser or a package. Elements hold children, attributes and -// listeners; text nodes hold text. `page(data, breakdowns)` runs diagnostics.js against -// the ids of index.html and returns what it drew. +// listeners; text nodes hold text. `page(data, breakdowns)` builds the elements of body.html +// that carry `data-sd`, runs diagnostics.js against them and returns what it drew. 'use strict'; const fs = require('node:fs'); @@ -10,6 +10,17 @@ const vm = require('node:vm'); const PAGE = path.join(__dirname, '..', '..', 'src', 'sentier_diagnostics', 'page'); +// the selectors the script asks for: [attr], [attr="value"], tag[attr], tag[attr="value"] +function matches(node, selector) { + const one = /^([a-z0-9]*)\[([a-z-]+)(?:="([^"]*)")?\]$/.exec(selector); + if (!one) throw new Error('the stand-in does not know the selector ' + selector); + const [, tag, key, value] = one; + if (node.text !== undefined) return false; + if (tag && node.tagName !== tag.toUpperCase()) return false; + if (!(key in node.attributes)) return false; + return value === undefined || node.attributes[key] === value; +} + class FakeNode { constructor(tag, text) { this.tagName = tag ? tag.toUpperCase() : null; @@ -22,7 +33,7 @@ class FakeNode { this.className = ''; this.value = ''; this.disabled = false; - this.onclick = null; + this.focused = false; } get firstChild() { return this.childNodes[0] || null; } @@ -60,23 +71,35 @@ class FakeNode { getAttribute(key) { return key in this.attributes ? this.attributes[key] : null; } + removeAttribute(key) { delete this.attributes[key]; } + addEventListener(type, listener) { (this.listeners[type] = this.listeners[type] || []).push(listener); } - dispatch(type, event) { (this.listeners[type] || []).forEach((listener) => listener({ target: this, ...event })); } + // an event on this node, passed up to its parents as a browser does + dispatch(type, event) { + const full = { target: this, ...event }; + for (let node = this; node; node = node.parentNode) (node.listeners[type] || []).forEach((listener) => listener(full)); + } closest(selector) { - if (selector !== 'tr[data-item]') throw new Error('the stand-in knows tr[data-item] only'); - for (let node = this; node; node = node.parentNode) { - if (node.tagName === 'TR' && node.getAttribute('data-item') !== null) return node; - } + for (let node = this; node; node = node.parentNode) if (matches(node, selector)) return node; return null; } + querySelectorAll(selector) { return this.all().slice(1).filter((n) => matches(n, selector)); } + + querySelector(selector) { return this.querySelectorAll(selector)[0] || null; } + + contains(other) { for (let n = other; n; n = n.parentNode) if (n === this) return true; return false; } + + focus() { this.focused = true; } + scrollIntoView() {} - // every text a reader sees: text nodes and tooltips, in order + // every text a reader sees: text nodes and tooltips, in order; nothing under a hidden node seen() { if (this.text !== undefined) return [this.text]; + if (this.hidden) return []; const own = this.attributes.title ? [this.attributes.title] : []; return own.concat(...this.childNodes.map((c) => c.seen())); } @@ -85,44 +108,78 @@ class FakeNode { all() { return this.text !== undefined ? [] : [this].concat(...this.childNodes.map((c) => c.all())); } } -function page(data, breakdowns) { - const html = fs.readFileSync(path.join(PAGE, 'index.html'), 'utf8'); - const byId = {}; +// the root of body.html and every element in it that carries `data-sd`, each a child of the +// root; the data scripts as the Python writes them where `scripts` holds them +function body(scripts) { + const html = fs.readFileSync(path.join(PAGE, 'body.html'), 'utf8'); + const root = new FakeNode('div'); + root.setAttribute('class', 'sentier-diagnostics'); + root.setAttribute('data-sentier-diagnostics', ''); for (const [whole, tag] of html.matchAll(/<([a-z0-9]+)\b[^>]*>/g)) { - const id = /\sid="([^"]+)"/.exec(whole); - if (!id) continue; - byId[id[1]] = new FakeNode(tag); - byId[id[1]].attributes.id = id[1]; - byId[id[1]].hidden = /\shidden[\s>=]/.test(whole); + const name = /\sdata-sd="([^"]+)"/.exec(whole); + if (!name) continue; + const node = new FakeNode(tag); + node.setAttribute('data-sd', name[1]); + node.hidden = /\shidden[\s>=]/.test(whole); + root.appendChild(node); + } + for (const [name, value] of Object.entries(scripts || {})) { + const node = new FakeNode('script'); + node.setAttribute('type', 'application/json'); + node.setAttribute('data-sd', name); + node.textContent = JSON.stringify(value); + root.appendChild(node); } + return root; +} + +// `options`: {standalone, attributes, globals, fetch, protocol, scripts} +function page(data, breakdowns, options) { + options = options || {}; + const scripts = options.scripts || {}; + if (data !== undefined && !options.globals) scripts.data = data; + if (breakdowns && !options.globals) scripts.breakdowns = breakdowns; + const root = body(scripts); + if (options.standalone) root.setAttribute('data-sd-standalone', ''); + for (const [key, value] of Object.entries(options.attributes || {})) root.setAttribute(key, value); + const head = new FakeNode('head'); const listeners = {}; const document = { title: '', readyState: 'complete', + head, + activeElement: null, createElement: (tag) => new FakeNode(tag), createTextNode: (text) => new FakeNode(null, String(text)), - getElementById: (id) => byId[id] || null, + querySelectorAll: (selector) => (matches(root, selector) ? [root] : []), addEventListener: () => {}, }; + root.ownerDocument = document; const context = { document, - location: { hash: '', search: '', pathname: '/index.html', protocol: 'file:' }, - history: { replaceState: () => {} }, + location: { hash: '', search: '', pathname: '/index.html', protocol: options.protocol || 'file:' }, + history: { replaceState: () => { context.location.hash = ''; } }, addEventListener: (type, listener) => { (listeners[type] = listeners[type] || []).push(listener); }, URLSearchParams, FileReader: class {}, - fetch: () => Promise.reject(new Error('no network')), - SENTIER_DIAGNOSTICS: data, - SENTIER_DIAGNOSTICS_BREAKDOWNS: breakdowns || {}, + fetch: options.fetch || (() => Promise.reject(new Error('no network'))), + ...(options.globals || {}), }; + if (options.globals && data !== undefined) context.SENTIER_DIAGNOSTICS = data; + if (options.globals && breakdowns) context.SENTIER_DIAGNOSTICS_BREAKDOWNS = breakdowns; context.window = context; vm.createContext(context); vm.runInContext(fs.readFileSync(path.join(PAGE, 'diagnostics.js'), 'utf8'), context, { filename: 'diagnostics.js' }); + const byName = {}; + root.childNodes.forEach((n) => { if (n.getAttribute('data-sd')) byName[n.getAttribute('data-sd')] = n; }); return { - byId, + root, + byId: byName, // by the name in data-sd + head, document, - // the text of everything drawn, tooltips included - text: () => Object.values(byId).filter((n) => !n.hidden).map((n) => n.seen().join(' ')).join('\n'), + context, + // the text of everything drawn and not hidden, tooltips included + text: () => root.childNodes.filter((n) => !n.hidden && n.tagName !== 'SCRIPT').map((n) => n.seen().join(' ')).join('\n'), open: (hash) => { context.location.hash = '#' + hash; (listeners.hashchange || []).forEach((listener) => listener({})); @@ -130,4 +187,4 @@ function page(data, breakdowns) { }; } -module.exports = { FakeNode, page }; +module.exports = { FakeNode, page, body }; diff --git a/tests/js/page.test.js b/tests/js/page.test.js index b03832c..ba8dbc8 100644 --- a/tests/js/page.test.js +++ b/tests/js/page.test.js @@ -1,5 +1,5 @@ // The page script against the acceptance cases the Python rule is tested on. -// Run with `node --test tests/js/`. Node's own test runner: no package, no install. +// Run with `node --test tests/js/page.test.js`. Node's own test runner: no package, no install. 'use strict'; const test = require('node:test'); @@ -140,7 +140,7 @@ const example = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'diagnostics const wheat = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'p01-fix-1.json'), 'utf8')); test('the example page is drawn whole', () => { - const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }); + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }, { standalone: true }); assert.equal(drawn.byId.load.hidden, true, drawn.byId['load-error'].textContent); assert.equal(drawn.byId.content.hidden, false); assert.equal(drawn.byId.unchecked.hidden, true); @@ -202,7 +202,7 @@ test('a text by a model is drawn as a claim nobody checked (A9)', () => { test('nothing drawn calls a move better or worse (A22)', () => { const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }); drawn.open('item=p01'); - const html = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'sentier_diagnostics', 'page', 'index.html'), 'utf8') + const html = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'sentier_diagnostics', 'page', 'body.html'), 'utf8') .replace(/<[^>]*>/g, ' '); const text = (drawn.text() + '\n' + html).toLowerCase(); for (const word of ['better', 'worse', 'improve', 'closer']) assert.ok(!text.includes(word), word); @@ -217,7 +217,7 @@ test('the item view splits only what can be split and draws the breakdown of its assert.ok(/100\.0 %/.test(body), 'the sum of the shares'); drawn.open('item=p03'); // barley: no water use in the rebuilt state const barley = drawn.byId['item-body']; - const water = barley.all().filter((n) => n.tagName === 'TABLE')[1]; + const water = barley.querySelector('table[data-indicator="x:water"]'); const shares = water.all().filter((n) => n.tagName === 'TR').slice(1, -1).map((tr) => tr.childNodes[5].textContent); assert.deepEqual(shares, shares.map(() => '—')); assert.ok(water.seen().join(' ').includes('not split: missing in a state between')); @@ -241,3 +241,169 @@ test('a file the page cannot draw draws nothing', () => { assert.equal(drawn.byId.load.hidden, false); assert.equal(drawn.byId.title.textContent, ''); }); + +// ------------------------------------------------------------------ the matrix and the drawer + +const local = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'local', 'diagnostics.json'), 'utf8')); +const full = () => JSON.parse(fs.readFileSync(path.join(EXAMPLE, 'local', 'matrix.json'), 'utf8')); +const gridRows = (drawn) => drawn.byId['grid-table'].querySelectorAll('tr[data-item]'); + +test('the band the page names for a matrix row is the band of the Python rule', () => { + for (const c of cases) { + if (c.class !== 'differs' && c.class !== 'beyond 10x') continue; + assert.equal(page.bandOf(c.before, c.after, c.class, c.direction), c.band, c.id); + } +}); + +test('a ratio never reads 1 when it is not, nor Infinity', () => { + assert.equal(page.ratioText(2, 3), '1.5'); + assert.equal(page.ratioText(-2, -4), '2'); + assert.equal(page.ratioText(1, 1.0000003), '1 + 3e−7'); + assert.equal(page.ratioText(1, 0.9999), '1 − 1e−4'); + assert.equal(page.ratioText(1e-300, 1e300), '1e600'); + assert.equal(page.ratioText(1e300, 1e-300), '1e−600'); +}); + +test('the matrix the Python writes is one the page takes, and the rows are the file\'s', () => { + const data = local(); + assert.equal(page.checkMatrix(full(), data), null); + const rows = page.matrixRows(full(), data); + assert.equal(rows.length, data.items_count.all); + for (const item of data.items) { + const row = rows.find((r) => r.id === item.id); + assert.deepEqual(row.values, item.values, item.id); + assert.deepEqual(row.classes, item.classes, item.id); + assert.deepEqual(row.directions, item.directions, item.id); + assert.deepEqual(row.verdicts, item.verdicts, item.id); + } +}); + +test('a matrix of another file, or of another shape, is refused', () => { + const data = local(); + const other = full(); other.of.title = 'Another'; + assert.match(page.checkMatrix(other, data), /another file/); + const short = full(); short.classes[0] = short.classes[0].slice(1); + assert.match(page.checkMatrix(short, data), /class code/); + const wrong = full(); wrong.values[2][0] = 'x'; + assert.match(page.checkMatrix(wrong, data), /not a number/); + const fewer = full(); fewer.items.pop(); fewer.values.pop(); fewer.classes.pop(); fewer.verdicts.pop(); + assert.match(page.checkMatrix(fewer, data), /says 10 items/); +}); + +test('the grid shows the listed items, says so, and every item once the matrix is here', () => { + const data = local(); + const before = dom.page(data); // over file: the matrix.json the file names cannot be read + assert.equal(gridRows(before).length, data.items.length); + assert.match(before.byId['matrix-said'].textContent, /not loaded: matrix\.json is read over http only/); + assert.match(before.byId['grid-said'].textContent, new RegExp('of ' + data.items.length + ' products, among the ' + data.items.length + ' products in this file; ' + data.items_count.not_listed + ' more')); + const after = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + assert.equal(gridRows(after).length, data.items_count.all); + assert.match(after.byId['matrix-said'].textContent, /matrix of all 10 products is loaded/); + assert.equal(after.byId['grid-table'].all().filter((n) => n.className === 'sd-listed').length, data.items.length); +}); + +test('a matrix beside the page is loaded as a script after the page is drawn', () => { + const drawn = dom.page(local(), undefined, { attributes: { 'data-sd-matrix': 'matrix.js', 'data-sd-base': 'runs/1/' } }); + const script = drawn.head.querySelector('script[src]'); + assert.equal(script.getAttribute('src'), 'runs/1/matrix.js'); + assert.match(drawn.byId['matrix-said'].textContent, /^Loading/); + drawn.context.SENTIER_DIAGNOSTICS_MATRIX = full(); + script.onload(); + assert.equal(gridRows(drawn).length, 10); +}); + +test('a matrix is fetched as JSON over http, and one of another file is not drawn', async () => { + const served = (m) => () => Promise.resolve({ ok: true, status: 200, text: () => Promise.resolve(JSON.stringify(m)) }); + const drawn = dom.page(local(), undefined, { protocol: 'https:', fetch: served(full()) }); + await new Promise((done) => setImmediate(done)); + assert.equal(gridRows(drawn).length, 10); + const other = full(); other.of.steps = other.of.steps.slice().reverse(); + const refused = dom.page(local(), undefined, { protocol: 'https:', fetch: served(other) }); + await new Promise((done) => setImmediate(done)); + assert.equal(gridRows(refused).length, 6); + assert.match(refused.byId['matrix-said'].textContent, /not loaded: it is the matrix of another file/); +}); + +test('a grid cell is the ratio after ÷ before, coloured by its direction', () => { + const drawn = dom.page(example()); + const wheat = gridRows(drawn).find((tr) => tr.getAttribute('data-item') === 'p01'); + const item = example().items.find((i) => i.id === 'p01'); + const values = item.values['x:climate']; + const cell = wheat.childNodes[3]; + assert.equal(item.classes['x:climate'][4], 'differs'); + assert.equal(cell.textContent, page.ratioText(values[0], values[values.length - 1])); + assert.match(cell.className, new RegExp('tone-' + item.directions['x:climate'][4])); +}); + +test('a heading sorts the grid by how far the two lie apart, the furthest first, and says so', () => { + const data = example(); + const drawn = dom.page(data); + const head = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]'); + const water = head.find((th) => th.getAttribute('title') && th.getAttribute('title').indexOf('Water') === 0); + water.querySelector('button[type="button"]').dispatch('click', {}); + const after = drawn.byId['grid-table'].querySelectorAll('th[scope="col"]').find((th) => th.getAttribute('aria-sort')); + assert.equal(after.getAttribute('aria-sort'), 'descending'); + const order = gridRows(drawn).map((tr) => tr.getAttribute('data-item')); + const key = (id) => { const i = data.items.find((x) => x.id === id); const v = i.values['x:water']; return page.severityKey(v[0], v[v.length - 1], i.classes['x:water'][4]); }; + for (let k = 1; k < order.length; k++) assert.ok(page.compareKeys(key(order[k - 1]), key(order[k])) <= 0, order.join()); +}); + +test('the grid draws 300 rows at a time', () => { + const data = local(); + const m = full(); + const n = 650; + const grown = { ...m, items: [], values: [], classes: [], verdicts: [] }; + for (let i = 0; i < n; i++) { + const at = i % m.items.length; + grown.items.push(i < m.items.length ? m.items[at] : [m.items[at][0] + '-' + i].concat(m.items[at].slice(1))); + grown.values.push(m.values[at]); grown.classes.push(m.classes[at]); grown.verdicts.push(m.verdicts[at]); + } + data.matrix.items = n; + const drawn = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: grown } }); + assert.equal(gridRows(drawn).length, 300); + assert.equal(drawn.byId['grid-more'].textContent, 'Show 300 more'); + drawn.byId['grid-more'].dispatch('click', {}); + drawn.byId['grid-more'].dispatch('click', {}); + assert.equal(gridRows(drawn).length, n); + assert.equal(drawn.byId['grid-more'].hidden, true); +}); + +test('the drawer opens on an item of the matrix, says why first, and closes with Escape', () => { + const data = local(); + const drawn = dom.page(data, undefined, { globals: { SENTIER_DIAGNOSTICS_MATRIX: full() } }); + const unlisted = full().items.map((r) => r[0]).find((id) => !data.items.some((i) => i.id === id)); + gridRows(drawn).find((tr) => tr.getAttribute('data-item') === unlisted).childNodes[0].dispatch('click', {}); + assert.equal(drawn.context.location.hash, 'item=' + unlisted); + drawn.open('item=' + unlisted); + assert.equal(drawn.byId.item.hidden, false); + assert.match(drawn.byId['item-meta'].textContent, /from the matrix: not listed in this file/); + assert.equal(drawn.byId['item-body'].childNodes[0].textContent, 'Why it differs'); + assert.equal(drawn.byId['item-title'].focused, true); + drawn.root.dispatch('keydown', { key: 'Escape' }); + assert.equal(drawn.byId.item.hidden, true); +}); + +test('why it differs: the item\'s own explanations, then those of a step or indicator it moved in, never a "none" over a part', () => { + const data = example(); + const drawn = dom.page(data); + for (const item of data.items) { + drawn.open('item=' + item.id); + const why = drawn.byId['item-body'].childNodes[1]; + const own = data.explanations.filter((e) => e.about.item === item.id).length; + const scopes = why.childNodes.map((li) => (li.childNodes[0].className === 'sd-scope' ? li.childNodes[0].textContent : 'none')); + assert.equal(scopes.filter((s) => s === 'This product').length, own, item.id); + assert.ok(scopes.indexOf('This product') <= 0 || scopes.slice(0, own).every((s) => s === 'This product'), item.id + ': its own first'); + if (scopes.includes('none')) assert.equal(why.childNodes.length, 1, item.id + ': "none" only where nothing is given'); + } +}); + +test('the page sets the title of a page of its own only, never a host\'s', () => { + assert.equal(dom.page(example()).document.title, ''); + assert.equal(dom.page(example(), undefined, { standalone: true }).document.title, example().title); +}); + +test('the data can come from a global, as a site\'s own data.js sets it', () => { + const drawn = dom.page(example(), { 'p01-fix-1.json': wheat() }, { globals: {} }); + assert.equal(drawn.byId.content.hidden, false); + assert.equal(drawn.byId.unchecked.hidden, true); +}); diff --git a/tests/test_matrix_page.py b/tests/test_matrix_page.py new file mode 100644 index 0000000..21d9b13 --- /dev/null +++ b/tests/test_matrix_page.py @@ -0,0 +1,315 @@ +"""The full matrix of a local file, and the page as a frameless body a site holds.""" + +from __future__ import annotations + +import contextlib +import io +import json +import re +import tempfile +import unittest +from html.parser import HTMLParser +from pathlib import Path + +from sentier_diagnostics import chain, cli, example, fileformat, matrix, page + +from . import helpers + + +def local(): + return example.build_local() + + +class Matrix(unittest.TestCase): + def test_the_local_example_and_its_matrix_agree(self): + data, full = local() + self.assertEqual(fileformat.errors(data), []) + self.assertEqual(matrix.errors(full), []) + self.assertEqual(matrix.against(full, data), []) + self.assertEqual(data["matrix"], {"file": "matrix.json", "items": 10}) + self.assertEqual(len(data["items"]), 6) + + def test_the_matrix_is_ordered_as_the_listed_items_are(self): + data, full = local() + self.assertEqual( + [r[0] for r in full["items"]][: len(data["items"])], [i["id"] for i in data["items"]] + ) + + def test_a_matrix_survives_its_own_text(self): + _, full = local() + self.assertEqual(json.loads(matrix.dumps(full)), full) + + def test_every_changed_code_is_found(self): + data, full = local() + cases = { + "a class code": lambda m: m["classes"].__setitem__(0, ("=" if m["classes"][0][0] != "=" else "D") + m["classes"][0][1:]), + "a direction": lambda m: m["classes"].__setitem__(0, m["classes"][0].swapcase()), + "a result": lambda m: m["values"][7].__setitem__(0, (m["values"][7][0] or 1.0) * 3), + "a verdict": lambda m: m["verdicts"].__setitem__( + 6, "".join("o" if c == "a" else "a" if c == "o" else c for c in m["verdicts"][6])), + "a name of a listed item": lambda m: m["items"][0].__setitem__(1, "Another name"), + "one item fewer": lambda m: [m[k].pop() for k in ("items", "values", "classes", "verdicts")], + "the title": lambda m: m["of"].__setitem__("title", "Another"), + } # fmt: skip + for name, change in cases.items(): + with self.subTest(name): + changed = helpers.mutated(full, change) + self.assertNotEqual(changed, full) + found = matrix.errors(changed) or matrix.against(changed, data) + self.assertNotEqual(found, [], name) + + def test_counts_that_are_not_the_matrix_are_found(self): + data, full = local() + cases = { + "a count": lambda d: d["counts"]["total"]["x:climate"]["classes"].__setitem__( + "agrees", d["counts"]["total"]["x:climate"]["classes"]["agrees"] + 1), + "the attribution": lambda d: d["attribution"]["x:climate"].__setitem__( + "spread", d["attribution"]["x:climate"]["spread"] + 1), + "the items worth a look": lambda d: d["items_count"].__setitem__("qualifying", d["items_count"]["qualifying"] - 1), + "the items of the matrix": lambda d: d["matrix"].__setitem__("items", 11), + } # fmt: skip + for name, change in cases.items(): + with self.subTest(name): + self.assertNotEqual(matrix.against(full, helpers.mutated(data, change)), [], name) + + def test_shape_faults_never_raise(self): + _, full = local() + for bad in (None, [], {"format": "sentier-diagnostics-matrix/2"}, {**full, "extra": 1}, + {**full, "items": [[1, 2, 3, 4, 5]]}, {**full, "classes": ["?" * 15] * 10}): # fmt: skip + with self.subTest(repr(bad)[:60]): + self.assertNotEqual(matrix.errors(bad), []) + + def test_a_matrix_is_written_and_read_back_checked(self): + data, full = local() + with tempfile.TemporaryDirectory() as folder: + where = matrix.write(full, Path(folder) / "matrix.json") + self.assertEqual(matrix.read(where, data), full) + other = helpers.mutated(data, lambda d: d.update(title="Another")) + with self.assertRaises(ValueError): + matrix.read(where, other) + + +class NamesAMatrix(unittest.TestCase): + """The key `matrix` of a diagnostics file (SPEC 4.2).""" + + def test_only_a_local_file_of_items_names_a_matrix(self): + data, _ = local() + public = helpers.mutated(data, lambda d: d.update(audience="public")) + self.assertTrue(any("only a local file" in e for e in fileformat.errors(public))) + + def test_the_matrix_holds_every_item(self): + data, _ = local() + wrong = helpers.mutated(data, lambda d: d["matrix"].update(items=9)) + self.assertNotEqual(fileformat.errors(wrong), []) + + def test_a_matrix_is_named_by_a_plain_file_name(self): + data, _ = local() + for name in ("../matrix.json", "matrix.js", "/abs.json", "a b.json"): + with self.subTest(name): + self.assertNotEqual( + fileformat.errors(helpers.mutated(data, lambda d: d["matrix"].update(file=name))), [] + ) + + def test_no_matrix_is_no_key_or_null(self): + data, _ = local() + self.assertEqual(fileformat.errors(helpers.mutated(data, lambda d: d.update(matrix=None))), []) + self.assertEqual(fileformat.errors(helpers.mutated(data, lambda d: d.pop("matrix"))), []) + + def test_a_public_file_is_never_built_with_a_matrix(self): + with self.assertRaises(ValueError): + example._made(chain.build_with_matrix, audience="public") + + +class Elements(HTMLParser): + """Every start tag of a fragment, with its attributes.""" + + def __init__(self): + super().__init__() + self.tags = [] + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + + +def tags_of(text: str) -> list[tuple[str, dict]]: + parser = Elements() + parser.feed(text) + return parser.tags + + +class Body(unittest.TestCase): + def test_one_root_and_no_frame_no_id(self): + text = page.body(example.build(), breakdowns={"p01-fix-1.json": example.wheat_breakdown()}) + tags = tags_of(text) + self.assertEqual(tags[0][0], "div") + self.assertEqual(tags[0][1].get("class"), "sentier-diagnostics") + self.assertIn("data-sentier-diagnostics", tags[0][1]) + self.assertEqual([t for t, a in tags if "data-sentier-diagnostics" in a], ["div"]) + self.assertFalse({"html", "head", "body", "main", "link", "style", "h1"} & {t for t, _ in tags}) + self.assertFalse([t for t, a in tags if "id" in a], "no id a host's page could hold twice") + self.assertTrue(text.rstrip().endswith("
      ")) + + def test_the_data_inside_is_the_file_and_cannot_end_its_element(self): + data = helpers.mutated(example.build(), lambda d: d.update(about="