From aecb0617c39be6eb5881545fefa4d96db2f86a76 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 05:04:16 +0000 Subject: [PATCH 001/126] chore(git): ignore generated web artifacts --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 4f3a6fb..091970c 100644 --- a/.gitignore +++ b/.gitignore @@ -212,5 +212,12 @@ output/ # CI smoke-test output ci-output/ +# Node.js / Next.js generated files +node_modules/ +.next/ +next-env.d.ts +*.tsbuildinfo +web/prisma/generated/ + # Stray git worktrees — never belong in the index .claude/worktrees/ From 74e9b0df8376b84c96e5ae9ff87039803cd5399c Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 05:04:16 +0000 Subject: [PATCH 002/126] chore(agents): configure repository workflows --- AGENTS.md | 13 +++++++++ docs/agents/domain.md | 51 ++++++++++++++++++++++++++++++++++++ docs/agents/issue-tracker.md | 30 +++++++++++++++++++++ docs/agents/triage-labels.md | 15 +++++++++++ 4 files changed, 109 insertions(+) create mode 100644 AGENTS.md create mode 100644 docs/agents/domain.md create mode 100644 docs/agents/issue-tracker.md create mode 100644 docs/agents/triage-labels.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4d00ed3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,13 @@ +## Agent skills + +### Issue tracker + +Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`. + +### Domain docs + +This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`. diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 0000000..3524904 --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,51 @@ +# Domain Docs + +How the engineering skills should consume this repo's domain documentation when exploring the codebase. + +## Before exploring, read these + +- **`CONTEXT.md`** at the repo root, or +- **`CONTEXT-MAP.md`** at the repo root if it exists: it points at one `CONTEXT.md` per context. Read each one relevant to the topic. +- **`docs/adr/`**: read ADRs that touch the area you're about to work in. In multi-context repos, also check `src//docs/adr/` for context-scoped decisions. + +If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved. + +## File structure + +Single-context repo (most repos): + +``` +/ +├── CONTEXT.md +├── docs/adr/ +│ ├── 0001-event-sourced-orders.md +│ └── 0002-postgres-for-write-model.md +└── src/ +``` + +Multi-context repo (presence of `CONTEXT-MAP.md` at the root): + +``` +/ +├── CONTEXT-MAP.md +├── docs/adr/ ← system-wide decisions +└── src/ + ├── ordering/ + │ ├── CONTEXT.md + │ └── docs/adr/ ← context-specific decisions + └── billing/ + ├── CONTEXT.md + └── docs/adr/ +``` + +## Use the glossary's vocabulary + +When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids. + +If the concept you need isn't in the glossary yet, that's a signal: either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`). + +## Flag ADR conflicts + +If your output contradicts an existing ADR, surface it explicitly rather than silently overriding: + +> _Contradicts ADR-0007 (event-sourced orders), but worth reopening because…_ diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 0000000..0209a19 --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,30 @@ +# Issue tracker: Local Markdown + +Issues and specs for this repo live as markdown files in `.scratch/`. + +## Conventions + +- One feature per directory: `.scratch//` +- The spec is `.scratch//spec.md` +- Implementation issues are one file per ticket at `.scratch//issues/-.md`, numbered from `01`, never a single combined tickets file +- Triage state is recorded as a `Status:` line near the top of each issue file (see `triage-labels.md` for the role strings) +- Comments and conversation history append to the bottom of the file under a `## Comments` heading + +## When a skill says "publish to the issue tracker" + +Create a new file under `.scratch//` (creating the directory if needed). + +## When a skill says "fetch the relevant ticket" + +Read the file at the referenced path. The user will normally pass the path or the issue number directly. + +## Wayfinding operations + +Used by `/wayfinder`. The **map** is a file with one **child** file per ticket. + +- **Map**: `.scratch//map.md` (the Notes / Decisions-so-far / Fog body). +- **Child ticket**: `.scratch//issues/NN-.md`, numbered from `01`, with the question in the body. A `Type:` line records the ticket type (`research`/`prototype`/`grilling`/`task`); a `Status:` line records `claimed`/`resolved`. +- **Blocking**: a `Blocked by: NN, NN` line near the top. A ticket is unblocked when every file it lists is `resolved`. +- **Frontier**: scan `.scratch//issues/` for files that are open, unblocked, and unclaimed; first by number wins. +- **Claim**: set `Status: claimed` and save before any work. +- **Resolve**: append the answer under an `## Answer` heading, set `Status: resolved`, then append a context pointer (gist + link) to the map's Decisions-so-far in `map.md`. diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 0000000..b716855 --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,15 @@ +# Triage Labels + +The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker. + +| Label in mattpocock/skills | Label in our tracker | Meaning | +| -------------------------- | -------------------- | ---------------------------------------- | +| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue | +| `needs-info` | `needs-info` | Waiting on reporter for more information | +| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent | +| `ready-for-human` | `ready-for-human` | Requires human implementation | +| `wontfix` | `wontfix` | Will not be actioned | + +When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table. + +Edit the right-hand column to match whatever vocabulary you actually use. From 21d5734ea9f7e89b7bd76ef0ab3a0c3eda1cccba Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 05:04:17 +0000 Subject: [PATCH 003/126] docs(evaluation): record domain and architecture decisions --- CONTEXT.md | 53 +++++++++++++++++++ docs/adr/0001-separate-review-from-scoring.md | 3 ++ ...0002-rank-models-by-workbook-row-recall.md | 3 ++ 3 files changed, 59 insertions(+) create mode 100644 CONTEXT.md create mode 100644 docs/adr/0001-separate-review-from-scoring.md create mode 100644 docs/adr/0002-rank-models-by-workbook-row-recall.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..3f3e2f1 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,53 @@ +# Accessibility Audit Evaluation + +This context describes the language used to evaluate AI-generated accessibility audit results against a human-reviewed reference. + +## Language + +**Reference defect**: +A human-reviewed accessibility problem derived from the mandated defect-report workbook and retained as benchmark evidence with traceable source provenance. +_Avoid_: Ground-truth row, Excel issue, expected finding + +**Audit finding**: +An accessibility problem reported by the AI audit model under evaluation. Programmatic checker output is not an audit finding for the primary model-efficacy score. +_Avoid_: Reference defect, programmatic finding, model response + +**Workbook-row recall**: +The count and proportion of scorable workbook source rows caught by correctly matched audit findings. It is the primary model-ranking metric; cost is the only tie-breaker. +_Avoid_: Precision, F1, aggregate quality score + +**LLM-eligible reference defect**: +A reference defect for which the AI audit model receives enough evidence and responsibility to make the relevant accessibility judgment. Defects intentionally handled only by programmatic checks are not LLM-eligible. +_Avoid_: Every workbook row, programmatic defect + +**Benchmark snapshot**: +An immutable capture of the audited webpage used as the common input for every evaluated model, with its source and capture identity recorded. +_Avoid_: Live page, test URL + +**Eligibility classification**: +The reviewed assignment explaining whether a reference defect belongs to the AI model's responsibility, a programmatic check, unavailable audit evidence, or unresolved ambiguity. +_Avoid_: Model score, exclusion without rationale + +**Programmatic finding**: +An accessibility problem reported by a deterministic checker rather than an AI audit model. It contributes to combined workbook coverage but not to model ranking. +_Avoid_: Audit finding, LLM-eligible reference defect + +**Combined workbook coverage**: +The count and proportion of in-scope workbook rows caught by either programmatic findings or audit findings, with unavailable and ambiguous rows still reported explicitly. +_Avoid_: Workbook-row recall, model score + +**Canonical audit finding**: +A lossless, structured representation of one raw audit finding used for comparison across prompts and models. +_Avoid_: Production report row, raw model response + +**Match decision**: +An approved determination that one canonical audit finding does or does not correspond to one reference defect, with supporting evidence and rationale. +_Avoid_: Candidate match, similarity score + +**Estimated run cost**: +The calculated cost of one audit run using API-reported token usage and a recorded model-price schedule. It is the sole tie-breaker when workbook-row recall is equal. +_Avoid_: Invoice amount, latency score + +**Comparison reviewer**: +A human or model that proposes eligibility classifications or match decisions through the same review process. A comparison reviewer does not calculate the final score. +_Avoid_: Scorer, audit model diff --git a/docs/adr/0001-separate-review-from-scoring.md b/docs/adr/0001-separate-review-from-scoring.md new file mode 100644 index 0000000..cdc2e48 --- /dev/null +++ b/docs/adr/0001-separate-review-from-scoring.md @@ -0,0 +1,3 @@ +# Separate review from deterministic scoring + +The evaluation framework separates eligibility and match review from scoring: `EligibilityReviewer` and `MatchReviewer` produce structured decisions through human-reviewed file adapters initially and optional model-backed adapters later, while the scorer consumes only approved decisions and remains deterministic. This keeps identical normalized inputs exactly reproducible, lets an Astra reviewer replace tedious human review without rewriting import, scoring, or reporting, and allows workbook-derived review artifacts to remain private even though authorized models may process them. diff --git a/docs/adr/0002-rank-models-by-workbook-row-recall.md b/docs/adr/0002-rank-models-by-workbook-row-recall.md new file mode 100644 index 0000000..ed4fd57 --- /dev/null +++ b/docs/adr/0002-rank-models-by-workbook-row-recall.md @@ -0,0 +1,3 @@ +# Rank models by workbook-row recall + +Models are ranked first by workbook-row recall over human-approved `llm_eligible` rows, with estimated audit-run cost as the only tie-breaker; latency, token usage, unmatched findings, programmatic coverage, and combined workbook coverage are reported but do not change rank. This keeps the mandated defect workbook as the principal measure while preventing model-independent programmatic findings or an arbitrary composite formula from obscuring the model comparison. From fe861a6fbd0ff0503a2a81e67d6ca9a28747b3e3 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 05:04:17 +0000 Subject: [PATCH 004/126] docs(evaluation): add model evaluation specification --- .scratch/model-evaluation/spec.md | 178 ++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .scratch/model-evaluation/spec.md diff --git a/.scratch/model-evaluation/spec.md b/.scratch/model-evaluation/spec.md new file mode 100644 index 0000000..658932d --- /dev/null +++ b/.scratch/model-evaluation/spec.md @@ -0,0 +1,178 @@ +Status: ready-for-agent + +## Problem Statement + +The project needs a trustworthy way to compare how effectively different AI models perform the repository's accessibility-auditing role, especially whether smaller, less expensive models find nearly as many known accessibility problems as larger models. The mandated source of reference problems is the Pristine Accessibility Defect Report workbook, whose prose, locations, and recommendations are not standardized enough for direct automated scoring. The current audit pipeline also combines deterministic checks with AI judgment, filters some programmatically detectable defects out of model inputs, emits heterogeneous prompt responses, and produces a final CSV that drops some active prompt categories and applies extra filtering. Without a separate evaluation framework, a model comparison would conflate model performance with deterministic findings, missing evidence, post-processing, page drift, and subjective matching. + +The immediate goal is not to prove that a small model is equivalent to a large model. It is to prove that the evaluation method can import the required workbook, identify which reference defects the AI model is responsible for finding, normalize raw audit findings without losing data, support reviewable one-to-one matching, calculate an exact repeatable score, and report efficacy and cost transparently. The first proof of concept will use GPT-5.6 Luna at medium reasoning against a frozen Pristine homepage snapshot. + +## Solution + +Build a private, CLI-only model-evaluation framework around the existing audit pipeline. The framework will preserve the workbook as the required source of truth while transforming its homepage rows into human-reviewed reference defects with explicit eligibility classifications. It will freeze and hash a Pristine homepage benchmark snapshot, run the existing programmatic and AI audit paths without changing prompts, normalize raw prompt responses into canonical audit findings, generate reviewable candidate matches, and calculate workbook-row recall from approved match decisions. + +The deterministic scorer will remain separate from review. Human-reviewed files will initially implement the eligibility and match reviewer interfaces; future Astra-backed adapters can produce the same decisions without changing import, scoring, or reporting. Models will be ranked first by workbook-row recall over approved LLM-eligible rows and, only when recall is equal, by estimated audit cost. Programmatic coverage, combined workbook coverage, latency, token usage, malformed output, and unmatched findings will remain visible but will not alter model rank. + +Private inputs and generated artifacts will stay outside version control. The committed repository will contain only the framework, schemas, documentation, versioned public pricing configuration, and synthetic tests. Live API execution will be impossible without an explicit live flag, a cost limit, an available key, and per-run authorization after a dry-run summary. + +## User Stories + +1. As an accessibility-audit researcher, I want to compare models against the mandated defect workbook, so that results satisfy the project's external reference-data requirement. +2. As an accessibility-audit researcher, I want the first proof of concept to validate the measurement method, so that a weak or strong Luna result is not confused with evaluator quality. +3. As a project lead, I want every scored result to trace back to a workbook source row, so that I can explain the benchmark to stakeholders. +4. As a benchmark curator, I want the workbook checksum recorded, so that an updated or substituted workbook cannot silently change results. +5. As a contributor, I want a clear error when the private workbook is absent, so that I know to obtain it from an authorized project source. +6. As a contributor, I want to override the default workbook location explicitly, so that local workflows are not tied to one machine layout. +7. As a data owner, I want the workbook excluded from version control, so that employer-provided information is not published without approval. +8. As a data owner, I want workbook-derived references, reviews, snapshots, runs, and reports excluded from version control, so that derived private information is protected too. +9. As a benchmark curator, I want each workbook row imported without semantic rewriting, so that the original evidence remains intact. +10. As a benchmark curator, I want each reference defect to retain its source sheet and row, so that normalization is auditable. +11. As a benchmark curator, I want each homepage and applicable Global row classified, so that no workbook row silently disappears. +12. As a benchmark curator, I want the allowed eligibility classifications validated, so that references consistently distinguish AI responsibility, programmatic responsibility, unavailable evidence, and ambiguity. +13. As an evaluator, I want LLM-eligible reference defects clearly separated from programmatic defects, so that models are scored only on work assigned to them. +14. As a project lead, I want unavailable-evidence rows reported with reasons, so that pipeline blind spots remain visible. +15. As a project lead, I want ambiguous rows excluded from the scoring denominator until approved, so that unresolved interpretation does not arbitrarily help or hurt a model. +16. As a comparison reviewer, I want eligibility proposals presented with workbook and pipeline evidence, so that I can approve classifications efficiently. +17. As a comparison reviewer, I want a private Excel review workbook, so that multiline evidence and decisions can be reviewed side by side. +18. As a comparison reviewer, I want accepted, rejected, and needs-review decisions, so that uncertainty is explicit. +19. As a comparison reviewer, I want reviewer identity, rationale, confidence, and timestamp retained, so that decisions have provenance. +20. As a future automation maintainer, I want human and model reviewers to satisfy the same interfaces, so that Astra can replace tedious review without changing the scorer. +21. As a future automation maintainer, I want eligibility review and match review to be separate interfaces, so that benchmark curation and per-run adjudication can evolve independently. +22. As a data owner, I want future Astra review to use the authorized API project without publishing inputs, so that automation does not weaken data privacy. +23. As an evaluator, I want a benchmark snapshot captured once from the Pristine homepage, so that every model sees identical page content. +24. As an evaluator, I want the snapshot URL, capture time, HTTP metadata, and SHA-256 recorded, so that the tested page state is identifiable. +25. As a project lead, I want the stakeholder assertion that Pristine has not changed since the workbook audit recorded with the snapshot, so that the temporal assumption is explicit. +26. As an evaluator, I want Global rows scored only for their homepage manifestation in this proof of concept, so that homepage evidence is not inflated into a site-wide claim. +27. As an evaluator, I want the existing programmatic audit run against the benchmark snapshot, so that deterministic workbook coverage and checker gaps are measured. +28. As an evaluator, I want programmatic findings excluded from the AI model score, so that model-independent results do not inflate similarity between models. +29. As a project lead, I want programmatic coverage reported separately, so that the value and gaps of deterministic checks are visible. +30. As a project lead, I want combined workbook coverage reported separately, so that I can see what the complete two-pass pipeline catches. +31. As an evaluator, I want the current prompts, filters, prompt order, and summary setting frozen, so that the proof of concept measures the current audit role rather than a redesigned prompt system. +32. As an evaluator, I want GPT-5.6 Luna called with medium reasoning and without inherited temperature sampling, so that the requested model configuration is explicit. +33. As an evaluator, I want all raw prompt responses saved, so that normalization and scoring can be audited later. +34. As an evaluator, I want every active prompt response normalized directly, so that categories omitted by the production CSV remain available for evaluation. +35. As an evaluator, I want malformed successful responses recorded rather than selectively retried, so that structured-output reliability is part of the observed run. +36. As an evaluator, I want each canonical audit finding to preserve its raw source, prompt, checklist, page, problem, location evidence, WCAG evidence, and parse status, so that comparison is explainable. +37. As a comparison reviewer, I want impossible finding/reference pairs removed deterministically, so that I do not review obviously incompatible matches. +38. As a comparison reviewer, I want plausible candidate pairs shown with evidence from both sides, so that matching decisions are efficient and defensible. +39. As an evaluator, I want a match to require compatible page scope, underlying accessibility failure, and location or element-group evidence, so that superficial wording similarity cannot earn credit. +40. As an evaluator, I want WCAG identifiers and fix recommendations treated as supporting evidence rather than exact-match requirements, so that equivalent findings with different labels can match. +41. As an evaluator, I want one-to-one matching enforced, so that one vague audit finding cannot claim several unrelated workbook rows. +42. As an evaluator, I want multi-element workbook rows treated as one scoring unit when they describe one grouped problem, so that the stakeholder-facing score remains row based. +43. As an evaluator, I want a row containing genuinely different required sub-defects credited only when all required parts match, so that partial detection is not overstated. +44. As an evaluator, I want identical approved normalized inputs to produce exactly the same score, so that scoring is deterministic. +45. As a project lead, I want workbook-row recall shown as caught rows, denominator, and percentage, so that model efficacy is easy to interpret. +46. As a project lead, I want estimated audit cost to be the only ranking tie-breaker, so that cheaper models win only when workbook-row recall is equal. +47. As a project lead, I want latency and token usage recorded but excluded from ranking, so that operational information does not override the agreed efficacy measure. +48. As a project lead, I want unmatched audit findings and structured-output failures reported without changing rank, so that model behavior remains visible without replacing the mandated metric. +49. As a budget owner, I want audit-model cost separated from evaluation-model cost, so that a future Astra reviewer does not distort the model-under-test comparison. +50. As a budget owner, I want total experiment cost reported separately, so that the full research expense remains visible. +51. As a budget owner, I want costs computed from API-reported token categories and a versioned price schedule, so that estimates can be reproduced after prices change. +52. As a budget owner, I want the unrounded estimated audit cost used for ties, so that display rounding cannot change rank. +53. As an operator, I want dry-run behavior by default even when an API key exists, so that environment configuration alone cannot spend money. +54. As an operator, I want live execution to require an explicit flag, so that spending is intentional. +55. As an operator, I want each summarized proof-of-concept run explicitly authorized, so that approval covers a known model, snapshot, request set, and configuration. +56. As an operator, I want a required maximum audit-cost guardrail, so that a live run stops before starting another request after exhausting its budget. +57. As an operator, I want bounded retries only for transient failures, so that recoverable provider errors do not automatically invalidate a run. +58. As an operator, I want every retry and any billed usage recorded, so that cost and reliability reports are complete. +59. As an evaluator, I want an exhausted transient failure to mark the run incomplete and unranked, so that infrastructure failure is not scored as model ignorance. +60. As an evaluator, I want successful malformed output to produce no usable findings for that prompt, so that model-format failure is represented honestly. +61. As an evaluator, I want end-to-end wall time and per-request duration recorded, so that latency can be analyzed without affecting rank. +62. As an evaluator, I want run manifests to contain content, configuration, code, prompt, parser, and pricing identities, so that only comparable runs are ranked together. +63. As an evaluator, I want incompatible benchmark versions rejected from ordinary comparison, so that changed inputs are not presented as model differences. +64. As a contributor, I want one CLI with prepare, audit, review, and report workflows, so that the evaluation lifecycle is discoverable. +65. As a contributor, I want the evaluation framework to reuse the production audit pipeline through an adapter, so that audit logic is not duplicated. +66. As a contributor, I want the scorer isolated from network and model dependencies, so that it is fast and deterministic to test. +67. As a reviewer, I want reports in JSON, CSV, and Markdown derived from one score object, so that machine, spreadsheet, and human views cannot disagree. +68. As a project lead, I want the homepage result labeled illustrative, so that three provisionally eligible rows are not used to claim broad model equivalence. +69. As a researcher, I want later benchmark expansion organized by page and template family, so that repetitive job pages do not overstate generalization. +70. As a maintainer, I want synthetic fixtures used in committed tests, so that CI never requires private workbook content. +71. As a maintainer, I want ordinary tests and CI to make no live provider calls, so that validation is free, safe, and repeatable. +72. As a maintainer, I want concise errors for missing approvals, incomplete runs, invalid decisions, checksum drift, and match conflicts, so that unsafe states cannot silently produce scores. + +## Implementation Decisions + +- The proof of concept is CLI-only. It will not add controls to the public website or audit API. +- A dedicated model-evaluation module will contain workbook import, benchmark preparation, canonical schemas, audit adaptation, normalization, review, matching, scoring, pricing, and reporting. A single evaluation entry point will expose `prepare`, `audit`, `review`, and `report` subcommands. +- The existing production pipeline remains the audit implementation. The evaluation module will call it through an adapter rather than copying audit logic. +- The external testing seam is the deterministic scorer: approved reference defects, canonical findings, programmatic findings, match decisions, and run metadata enter; one complete score object exits. All reports are projections of that object. +- Two new reviewer seams are required because two adapters are known: human-reviewed local files initially and model-backed review later. `EligibilityReviewer` handles benchmark eligibility decisions; `MatchReviewer` handles per-run match decisions. The scorer does not know which adapter produced an approved decision. +- Human review will use a private Excel workbook with separate Eligibility and Matches sheets. Reviewer adapters emit validated decisions with `accepted`, `rejected`, or `needs_review` state plus identity, rationale, confidence, and timestamp. +- The future Astra adapters will use the same reviewer interfaces and an internal shared OpenAI client. They are designed but disabled for the proof of concept. Workbook-derived data may be sent through the authorized API project, but may not be committed publicly without approval. +- The private source workbook remains at the repository root under its current name by default and is ignored by exact filename. A command-line override permits another local location. The importer validates a recorded SHA-256 and fails clearly on absence or drift. +- All private derived artifacts live under one ignored evaluation workspace partitioned into references, reviews, snapshots, runs, and reports. Only code, schemas, documentation, synthetic fixtures, and public pricing data are committed. +- `openpyxl` is added as a runtime dependency for workbook import and private review workbooks. `pytest` is added as a development dependency. +- The Pristine homepage is captured as raw HTML because that is the production pipeline's input. The capture is immutable and accompanied by source URL, retrieval time, HTTP metadata, SHA-256, and a note that stakeholders report no change since the workbook audit. +- The existing DAT Vision Aid HTML fixture is unrelated to the workbook and is never used for workbook-based scoring. +- The proof of concept includes workbook rows scoped as Home and Global on the Pristine homepage. Global rows are labeled as global claims supported by homepage evidence and are not multiplied across pages. +- Every source row becomes one reviewed reference record. Raw text and provenance are retained. Normalized fields include the canonical problem, location or element group, WCAG evidence, eligibility classification, rationale, approval state, and any required sub-defects. +- Eligibility classifications are exactly `llm_eligible`, `programmatic`, `unavailable_evidence`, and `ambiguous`. Only approved `llm_eligible` rows enter workbook-row recall. All classifications remain present in coverage reports. +- The programmatic audit runs first. Its findings are normalized and matched to reference defects through the same evidence-based review process. A programmatic classification does not imply that the checker succeeded; uncovered checker responsibilities are reported as gaps. +- The AI audit preserves current filtering, prompt order, sequential execution, disabled summary prompts, and the 8,192-token output limit. The proof of concept uses GPT-5.6 Luna with medium reasoning, Chat Completions, and no temperature parameter. +- Current audit prompts are unchanged and versioned by content hash. Structured-output enforcement is not enabled in the proof of concept because it would change the audited conditions. +- Raw prompt outputs, request metadata, usage, durations, stop reasons, and failures are saved before normalization. +- Canonical audit findings are built directly from raw prompt responses using fixed per-prompt parsers. The production final CSV, heuristic false-positive suppression, and model-based deduplication are not scoring inputs. +- A canonical audit finding preserves run and model identity, prompt and checklist, page, problem family and statement, element and location evidence, WCAG evidence, raw source finding, and parse status. +- Candidate generation may remove only hard incompatibilities such as different pages or incompatible evidence categories. It does not award matches. +- An approved match requires compatible page or declared scope, the same underlying accessibility failure, and compatible location, element, or element group. WCAG identifiers and remediation wording are supporting evidence. +- Matching is one-to-one at the normalized defect level. Conflicting accepted decisions fail validation rather than being resolved silently. +- One workbook source row remains one headline scoring point. A grouped row is caught when the same grouped defect is identified at the compatible location. A row with distinct required sub-defects is caught only when all required parts are approved as matched. +- The scorer is deterministic and performs no model or network call. Identical normalized inputs and approved decisions produce an identical score object. +- The primary ranking metric is workbook-row recall over approved LLM-eligible rows. Estimated audit cost is the only tie-breaker. No weighted composite score is produced. +- Programmatic coverage, combined workbook coverage, unmatched findings, parse failures, token usage, and latency are reported separately and do not affect model rank. +- Cost reporting separates audit cost, evaluation cost, and total experiment cost. Only audit cost affects ties. Costs use API-reported token categories and a versioned public price schedule; estimates are not represented as invoice totals. +- Latency reporting includes end-to-end audit wall time, per-request durations, and their sum. Latency never affects ranking. +- Evaluation execution is dry by default. An API key in the environment is insufficient to trigger a request. Live execution requires an explicit live flag and a maximum audit-cost limit. +- The proof-of-concept runner permits at most two additional attempts for timeouts, rate limits, and server errors, using bounded exponential backoff. Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. +- Every attempt is recorded. Transient failures exhausted after retries make the run incomplete and unranked. A successful malformed response remains part of the run, contributes no usable findings for that prompt, and is reported as a format failure. +- A run manifest records workbook and snapshot hashes, reference-set version, source-control commit and dirty state, prompt and parser hashes, filters, summaries, model, reasoning effort, endpoint, output limit, retry policy, usage, price-schedule version, timestamps, and run ID. +- Ordinary comparison rejects runs whose snapshot, reference set, prompts, or eligibility set differ. Cross-version analysis must be requested explicitly and labeled accordingly. +- Reports are emitted as JSON, CSV, and Markdown from one score object. The row-level view exposes eligibility, rationale, programmatic outcome, model match, evidence, and final disposition for every in-scope workbook row. +- The proof of concept proceeds through gates: implementation and tests; snapshot and reference preparation; human eligibility approval; dry-run summary; explicit live-run authorization; Luna audit; human match approval; final report. +- One authorization covers the complete summarized live run, including policy-compliant automatic retries. A materially different configuration or separate rerun requires new authorization. +- The homepage result is illustrative because the provisional inventory contains only three clearly LLM-eligible rows out of sixteen in-scope rows. Benchmark expansion is required before any small-versus-large model claim. + +## Testing Decisions + +- Tests assert external behavior at module interfaces, not private helper functions or implementation order. The principal test seam is the deterministic scorer's input bundle and returned score object because it exercises eligibility, one-to-one matching, ranking data, coverage, and report inputs at the highest stable seam. +- Workbook-import tests use synthetic Excel fixtures and assert provenance retention, homepage filtering, source-row identity, checksum validation, exact allowed eligibility states, and useful missing-file or drift errors. +- Snapshot tests use local HTTP fakes or saved synthetic responses and assert immutable content, provenance fields, and hashing without accessing the live Pristine site. +- Audit-adapter tests replace the production pipeline with a fake at its existing callable seam. They assert frozen configuration, raw-result preservation, manifest metadata, dry-run safety, live-flag requirements, cost-limit behavior, and retry classification without provider calls. +- Per-prompt normalization is tested through the normalization module's public interface using synthetic raw responses for every active non-summary prompt. Tests include valid arrays and objects, fenced JSON, empty results, malformed successful output, unknown prompt types, and all six categories omitted by the production CSV. +- Reviewer-contract tests run the same decision examples through human-file and fake-model adapters. They assert identical validated outputs, required provenance, allowed states, and rejection of incomplete decisions. +- Candidate and match tests assert that hard incompatibilities are removed, plausible pairs remain reviewable, one-to-one conflicts fail validation, grouped rows receive one point, and distinct required sub-defects require complete coverage. +- Scorer tests assert exact repeatability, correct denominators, binary row credit, separation of programmatic and AI coverage, exclusion of ambiguous and unavailable rows from LLM recall, and cost-only tie-breaking. +- Cost tests use fixed synthetic usage and price schedules. They assert input, cached-input, output, retry, audit, evaluation, and total experiment calculations without depending on current external pricing. +- Failure-policy tests distinguish transient transport failures, exhausted retries, authentication errors, invalid requests, and malformed successful responses. They assert that incomplete runs cannot be ranked. +- Reporting tests construct one score object and assert that JSON, CSV, and Markdown totals and row dispositions agree. +- CLI tests exercise prepare, audit, review, and report workflows against a temporary private workspace with fake adapters. They assert safe defaults, explicit live requirements, and actionable errors. +- Regression fixtures contain no employer workbook text, Pristine content, secrets, or provider responses copied from live runs. +- No ordinary test or CI task may require an API key, access the public internet, or spend money. +- Prior art is the repository's network-free API regression style and CI dry-run smoke path; the evaluation suite extends that philosophy with pytest and synthetic fixtures. + +## Out of Scope + +- Claiming that GPT-5.6 Luna performs as well as a larger model based on the homepage proof of concept. +- Ranking multiple models during the initial no-spend implementation and eligibility-review stage. +- Repeated audit runs, statistical variance analysis, confidence intervals, or stability scoring. +- Using precision, F1, latency, token count, or unmatched-finding count to rank models. +- Automatically accepting Astra eligibility or match decisions during the proof of concept. +- Calling Astra as a reviewer before the human-reviewed process has supplied calibration examples. +- Publishing the source workbook, normalized private reference data, Pristine snapshot, review decisions, raw runs, or reports to Git. +- Redesigning production prompts, filters, payload extractors, programmatic rules, final CSV generation, false-positive suppression, or deduplication. +- Enforcing a new unified structured-output schema on the audit model during the proof of concept. +- Adding evaluation features to the public website or HTTP API. +- Browser rendering, JavaScript execution, screenshots, interactive-state testing, or Chrome-plus-JAWS automation. +- Treating a current live capture as independently proven byte-identical to the September audit state; the benchmark instead records the stakeholder-supported assumption. +- Treating Global rows as verified on every site page. +- Expanding immediately to all seventeen workbook URLs or using repeated job-page rows to claim generalization. +- Computing an exact invoiced dollar amount or accounting for private contractual discounts unavailable from API usage metadata. +- Making a live provider request before eligibility approval, dry-run review, explicit authorization, and a cost limit. + +## Further Notes + +- The provisional homepage inventory contains sixteen workbook rows: five programmatic, three LLM-eligible, six unavailable to the current pipeline, and two ambiguous. These are draft classifications for human review, not accepted benchmark truth. +- Provisional LLM-eligible examples include decorative-image alt judgment and heading-quality problems. Programmatic examples include missing landmarks, missing skip navigation, actionable empty alt, and multiple H1 elements. Unavailable-evidence examples include visual list semantics, styled text that should be headings, carousel controls, Lottie animation, decorative literal glyphs, and media operability. Ambiguities include fragmented adjacent links and a workbook alt-text claim that may conflict with the page. +- The existing committed HTML fixture represents DAT Vision Aid, not Pristine, and cannot be used for this benchmark. +- The workbook is the mandatory provenance source, but approved normalization may mark rows unavailable or ambiguous and may represent distinct required sub-defects. These transformations must remain traceable and reviewable. +- The first implementation milestone ends at the eligibility-review gate and incurs no API cost. The cost limit for the first live Luna run will be selected only after the dry-run summary exposes the request set and estimated usage. +- Future benchmark expansion should sample distinct pages and template families rather than random rows because repeated job-page patterns dominate the workbook. From 9cc5b95c91ef0bcd473addcf5aa9b789d21b3cf6 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 05:04:51 +0000 Subject: [PATCH 005/126] docs(evaluation): add implementation ticket graph --- .../01-explicit-audit-request-configuration.md | 12 ++++++++++++ .../issues/02-private-evaluation-workspace.md | 14 ++++++++++++++ .../03-freeze-pristine-benchmark-snapshot.md | 13 +++++++++++++ .../issues/04-homepage-reference-set.md | 16 ++++++++++++++++ .../05-review-and-score-synthetic-comparison.md | 16 ++++++++++++++++ .../issues/06-programmatic-workbook-coverage.md | 15 +++++++++++++++ .../issues/07-safe-luna-dry-run.md | 14 ++++++++++++++ .../issues/08-authorized-live-audit.md | 16 ++++++++++++++++ .../issues/09-normalize-raw-audit-findings.md | 14 ++++++++++++++ .../issues/10-review-finding-matches.md | 15 +++++++++++++++ .../11-verify-complete-evaluation-workflow.md | 16 ++++++++++++++++ .../issues/12-run-luna-homepage-poc.md | 16 ++++++++++++++++ 12 files changed, 177 insertions(+) create mode 100644 .scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md create mode 100644 .scratch/model-evaluation/issues/02-private-evaluation-workspace.md create mode 100644 .scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md create mode 100644 .scratch/model-evaluation/issues/04-homepage-reference-set.md create mode 100644 .scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md create mode 100644 .scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md create mode 100644 .scratch/model-evaluation/issues/07-safe-luna-dry-run.md create mode 100644 .scratch/model-evaluation/issues/08-authorized-live-audit.md create mode 100644 .scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md create mode 100644 .scratch/model-evaluation/issues/10-review-finding-matches.md create mode 100644 .scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md create mode 100644 .scratch/model-evaluation/issues/12-run-luna-homepage-poc.md diff --git a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md new file mode 100644 index 0000000..2e404ac --- /dev/null +++ b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md @@ -0,0 +1,12 @@ +# 01: Make audit request configuration explicit + +**What to build:** Add a backward-compatible audit request seam that preserves current production behavior while allowing evaluation runs to specify model, reasoning effort, sampling omissions, output limit, and complete usage metadata. + +**Blocked by:** None (can start immediately). + +**Status:** ready-for-agent + +- [ ] Existing Anthropic, OpenAI, and Gemini production callers retain their current effective defaults. +- [ ] An OpenAI evaluation caller can explicitly request GPT-5.6 Luna with medium reasoning and no temperature parameter. +- [ ] The request result exposes the provider metadata, token categories, stop reason, duration, and failure information required by evaluation runs. +- [ ] Network-free tests verify legacy defaults and the Luna request shape through fakes. diff --git a/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md b/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md new file mode 100644 index 0000000..bc7bcca --- /dev/null +++ b/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md @@ -0,0 +1,14 @@ +# 02: Bootstrap a private, safe evaluation workspace + +**What to build:** Provide the CLI shell and private workspace conventions needed to prepare, run, review, and report evaluations without exposing employer data or accidentally spending money. + +**Blocked by:** None (can start immediately). + +**Status:** ready-for-agent + +- [ ] The evaluation CLI exposes discoverable prepare, audit, review, and report workflows. +- [ ] The source workbook is ignored by exact filename, and all generated references, reviews, snapshots, runs, and reports are ignored under one private workspace. +- [ ] Missing private inputs produce actionable instructions rather than downloads or silent substitutions. +- [ ] Dry-run behavior is the default even when an API key is present. +- [ ] Excel support is added as a runtime dependency and pytest as a development dependency, with lock and generated dependency artifacts kept consistent. +- [ ] Synthetic CLI tests prove workspace initialization and safe defaults without private data or network access. diff --git a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md new file mode 100644 index 0000000..ed8a497 --- /dev/null +++ b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md @@ -0,0 +1,13 @@ +# 03: Freeze a reproducible Pristine benchmark snapshot + +**What to build:** Add a preparation workflow that captures the raw Pristine homepage once and turns it into an immutable benchmark snapshot with enough provenance to reuse across model runs. + +**Blocked by:** 02: Bootstrap a private, safe evaluation workspace. + +**Status:** ready-for-agent + +- [ ] The preparation workflow saves the exact raw HTML consumed by the existing audit pipeline. +- [ ] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, SHA-256, and the stakeholder-supported temporal assumption. +- [ ] Reusing a snapshot does not refetch or silently overwrite it. +- [ ] The existing unrelated DAT Vision Aid fixture is rejected as a Pristine benchmark input. +- [ ] Tests use local HTTP fakes and verify content identity, provenance, hashing, and overwrite protection. diff --git a/.scratch/model-evaluation/issues/04-homepage-reference-set.md b/.scratch/model-evaluation/issues/04-homepage-reference-set.md new file mode 100644 index 0000000..5ea2ecb --- /dev/null +++ b/.scratch/model-evaluation/issues/04-homepage-reference-set.md @@ -0,0 +1,16 @@ +# 04: Generate and validate the homepage reference set + +**What to build:** Transform the mandated workbook's Home and applicable Global rows into a traceable, human-reviewed reference set and generate the private Eligibility review needed to reach the no-spend approval gate. + +**Blocked by:** 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot. + +**Status:** ready-for-agent + +- [ ] Workbook absence and SHA-256 drift fail with clear acquisition or validation guidance. +- [ ] Imported records retain workbook filename, sheet, source row, raw evidence, canonical URL, page scope, and WCAG evidence without semantic rewriting. +- [ ] The reference set includes every homepage and applicable Global row exactly once. +- [ ] Eligibility is limited to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with rationale and approval state required. +- [ ] Global rows are represented as global claims with homepage evidence rather than multiplied across pages. +- [ ] A private Eligibility workbook presents source evidence, proposed classification, rationale, approval, reviewer identity, confidence, timestamp, and notes. +- [ ] Human-reviewed decisions round-trip into a validated, versioned reference set through the `EligibilityReviewer` interface. +- [ ] Synthetic workbook tests cover provenance, filtering, review import, invalid decisions, and checksum validation. diff --git a/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md b/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md new file mode 100644 index 0000000..560ce53 --- /dev/null +++ b/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md @@ -0,0 +1,16 @@ +# 05: Review and score a synthetic comparison + +**What to build:** Demonstrate the complete deterministic review-to-report path with synthetic reference defects and findings, establishing the scoring seam before connecting real audit outputs. + +**Blocked by:** 04: Generate and validate the homepage reference set. + +**Status:** ready-for-agent + +- [ ] The `MatchReviewer` interface accepts human-reviewed decisions without coupling the scorer to Excel or future model adapters. +- [ ] Match decisions support accepted, rejected, and needs-review states with reviewer provenance, rationale, confidence, and timestamp. +- [ ] Validation enforces one-to-one matches and rejects conflicting accepted decisions. +- [ ] Grouped rows receive one point when their grouped defect is matched, while rows with distinct required sub-defects require all parts. +- [ ] The deterministic scorer calculates workbook-row recall, programmatic coverage, combined workbook coverage, unmatched findings, and parse failures from approved inputs. +- [ ] Estimated audit cost is the sole ranking tie-breaker; latency, token usage, and evaluation cost remain informational. +- [ ] JSON, CSV, and Markdown reports derive from one score object and agree on every total and row disposition. +- [ ] Repeated scoring of identical inputs produces byte-equivalent structured results apart from deliberately excluded presentation timestamps. diff --git a/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md b/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md new file mode 100644 index 0000000..396223a --- /dev/null +++ b/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md @@ -0,0 +1,15 @@ +# 06: Measure programmatic workbook coverage + +**What to build:** Run the existing deterministic accessibility checks against the benchmark snapshot and show which workbook rows they actually cover, separately from model efficacy. + +**Blocked by:** 04: Generate and validate the homepage reference set; 05: Review and score a synthetic comparison. + +**Status:** ready-for-agent + +- [ ] Existing semantic, form, and non-text programmatic checks run against the frozen snapshot without duplicated audit logic. +- [ ] Programmatic findings are normalized with stable page and location evidence while retaining their raw source. +- [ ] Reference matches are reviewable and use the same accepted-decision validation as other matches. +- [ ] A `programmatic` eligibility classification does not automatically award credit when the checker misses the row. +- [ ] Reports identify covered programmatic rows, uncovered checker responsibilities, unavailable evidence, and ambiguity. +- [ ] Programmatic results contribute to combined workbook coverage but never to model ranking. +- [ ] Tests exercise the complete programmatic path with synthetic HTML and approved match decisions. diff --git a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md new file mode 100644 index 0000000..49c1252 --- /dev/null +++ b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md @@ -0,0 +1,14 @@ +# 07: Prepare a safe Luna dry run + +**What to build:** Produce a no-cost Luna audit plan against the approved benchmark configuration, with complete provenance and an explicit summary suitable for spending authorization. + +**Blocked by:** 01: Make audit request configuration explicit; 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot; 04: Generate and validate the homepage reference set. + +**Status:** ready-for-agent + +- [ ] The dry run uses the frozen prompts, filters, prompt order, disabled summaries, sequential execution, and existing output limit. +- [ ] The planned configuration records GPT-5.6 Luna, medium reasoning, Chat Completions, and omitted temperature. +- [ ] Prompt payloads, request count, estimated input tokens, prompt hashes, parser hashes, and repository state are saved without making a provider call. +- [ ] The manifest records workbook, snapshot, reference-set, benchmark, configuration, and pricing identities. +- [ ] The command refuses live execution when the live flag, API key, approved reference set, or maximum audit-cost limit is absent. +- [ ] Tests prove that an environment key alone cannot trigger spending. diff --git a/.scratch/model-evaluation/issues/08-authorized-live-audit.md b/.scratch/model-evaluation/issues/08-authorized-live-audit.md new file mode 100644 index 0000000..144f88c --- /dev/null +++ b/.scratch/model-evaluation/issues/08-authorized-live-audit.md @@ -0,0 +1,16 @@ +# 08: Execute and record an authorized live audit + +**What to build:** Add the guarded live-audit path that executes an explicitly authorized Luna run, preserves every raw outcome, and accounts for reliability, usage, cost, and latency. + +**Blocked by:** 01: Make audit request configuration explicit; 07: Prepare a safe Luna dry run. + +**Status:** ready-for-agent + +- [ ] Live execution requires the explicit live flag, API key, maximum audit-cost limit, and the exact summarized benchmark configuration. +- [ ] Audit prompts run sequentially with medium reasoning, no temperature, disabled summaries, and the existing output limit. +- [ ] At most two additional attempts occur for timeouts, rate limits, and server errors using bounded exponential backoff. +- [ ] Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. +- [ ] Every attempt records request identity, response metadata, usage, duration, stop reason, and failure classification without exposing secrets. +- [ ] Billed retry usage contributes to audit cost, and the runner stops before starting another request once the cost guardrail is exhausted. +- [ ] Exhausted transient failures mark the run incomplete and unranked; malformed successful responses remain recorded as model-format failures. +- [ ] Fake-provider tests exercise success, retry, budget, malformed response, and incomplete-run behavior without spending money. diff --git a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md new file mode 100644 index 0000000..6295696 --- /dev/null +++ b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md @@ -0,0 +1,14 @@ +# 09: Normalize every raw audit prompt response + +**What to build:** Convert saved raw audit responses into lossless canonical audit findings without relying on the production CSV, suppression rules, or model-based deduplication. + +**Blocked by:** 08: Execute and record an authorized live audit. + +**Status:** ready-for-agent + +- [ ] Every active non-summary prompt has a fixed normalizer, including categories omitted by the production CSV. +- [ ] Canonical audit findings retain run and model identity, prompt, checklist, page, problem family and statement, location evidence, WCAG evidence, raw source, and parse status. +- [ ] Valid object, array, fenced JSON, empty, malformed, and unknown prompt responses produce explicit deterministic outcomes. +- [ ] Successful malformed responses produce no usable findings and remain visible as format failures. +- [ ] Normalization performs no model or network call and applies no heuristic suppression or deduplication. +- [ ] Synthetic fixtures cover every prompt normalizer without copying private or live provider data. diff --git a/.scratch/model-evaluation/issues/10-review-finding-matches.md b/.scratch/model-evaluation/issues/10-review-finding-matches.md new file mode 100644 index 0000000..0c352e7 --- /dev/null +++ b/.scratch/model-evaluation/issues/10-review-finding-matches.md @@ -0,0 +1,15 @@ +# 10: Generate and validate finding matches + +**What to build:** Turn canonical audit findings and approved reference defects into an efficient, evidence-rich human match review whose accepted decisions can be scored deterministically. + +**Blocked by:** 04: Generate and validate the homepage reference set; 09: Normalize every raw audit prompt response. + +**Status:** ready-for-agent + +- [ ] Candidate generation removes only hard incompatibilities such as different pages or incompatible evidence categories. +- [ ] Candidate generation never awards credit or makes a semantic match decision. +- [ ] A private Matches workbook presents reference evidence, audit evidence, proposed disposition, rationale, confidence, approval, reviewer identity, timestamp, and notes side by side. +- [ ] An accepted match requires compatible page or scope, the same underlying accessibility failure, and compatible location, element, or element group. +- [ ] WCAG identifiers and remediation wording remain supporting evidence rather than exact-match requirements. +- [ ] Approved decisions round-trip through the `MatchReviewer` interface and enforce one-to-one matching. +- [ ] Conflicts, unresolved decisions, incomplete grouped evidence, and missing required sub-defects fail clearly rather than being resolved silently. diff --git a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md new file mode 100644 index 0000000..68885a4 --- /dev/null +++ b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md @@ -0,0 +1,16 @@ +# 11: Verify the complete evaluation workflow + +**What to build:** Connect preparation, audit, review, scoring, and reporting into one coherent workflow and prove the entire lifecycle with synthetic provider data before any real model run. + +**Blocked by:** 05: Review and score a synthetic comparison; 06: Measure programmatic workbook coverage; 08: Execute and record an authorized live audit; 10: Generate and validate finding matches. + +**Status:** ready-for-agent + +- [ ] The CLI completes prepare, audit, review, and report workflows against a temporary private workspace using synthetic inputs. +- [ ] Ordinary comparisons reject mismatched snapshots, reference sets, prompts, parsers, eligibility decisions, or benchmark configurations. +- [ ] Run manifests contain content identity, code state, request configuration, retry policy, usage, pricing, timestamps, and run identity. +- [ ] Audit, evaluation, and total experiment costs remain separate, and only audit cost affects equal-recall ranking. +- [ ] End-to-end and per-request latency are reported but never ranked. +- [ ] JSON, CSV, and Markdown reports expose every in-scope workbook row and agree on scoring and coverage. +- [ ] CI runs the pytest suite without private workbook data, public network access, provider credentials, or spending. +- [ ] Documentation explains the private-data workflow, human approval gates, dry-run review, live authorization, and illustrative limits of the homepage benchmark. diff --git a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md new file mode 100644 index 0000000..285288f --- /dev/null +++ b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md @@ -0,0 +1,16 @@ +# 12: Run the gated Luna homepage proof of concept + +**What to build:** Execute the approved operational workflow on the private Pristine benchmark and produce the first illustrative Luna evaluation report without bypassing any human or spending gate. + +**Blocked by:** 11: Verify the complete evaluation workflow. + +**Status:** ready-for-agent + +- [ ] The private workbook checksum, snapshot, reference set, and eligibility classifications are reviewed and approved before the audit proceeds. +- [ ] A no-cost dry-run summary presents the exact Luna configuration, request set, estimated usage, and proposed cost limit. +- [ ] The live run begins only after explicit user authorization covering that summarized configuration and cost guardrail. +- [ ] The authorized Luna run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. +- [ ] Canonical findings and candidate matches are generated without production CSV filtering or model-based deduplication. +- [ ] Ambiguous match decisions receive human approval before scoring. +- [ ] The final reports show Luna workbook-row recall, programmatic coverage, combined workbook coverage, estimated audit cost, tokens, latency, unmatched findings, and format failures. +- [ ] The result is labeled illustrative and makes no small-versus-large model claim from the homepage's limited eligible denominator. From c37197983579df1183607b657a90fc031c7bee2c Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 14:15:02 +0000 Subject: [PATCH 006/126] feat(audit): add explicit request configuration --- entry_points/api_server.py | 5 + entry_points/run_pipeline.py | 142 +-------- processing_scripts/llm_client/__init__.py | 3 + processing_scripts/llm_client/audit.py | 347 ++++++++++++++++++++++ tests/test_audit_request.py | 336 +++++++++++++++++++++ 5 files changed, 707 insertions(+), 126 deletions(-) create mode 100644 processing_scripts/llm_client/audit.py create mode 100644 tests/test_audit_request.py diff --git a/entry_points/api_server.py b/entry_points/api_server.py index 0f61b4f..7727ede 100644 --- a/entry_points/api_server.py +++ b/entry_points/api_server.py @@ -150,9 +150,14 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non "wcag_criteria": data.get("wcag_criteria", []), "status": status, "error": (api_result or {}).get("error"), + "failure": (api_result or {}).get("failure"), "parsed": parsed, + "provider": (api_result or {}).get("provider"), + "request": (api_result or {}).get("request"), + "usage": usage, "input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens"), + "stop_reason": (api_result or {}).get("stop_reason"), "duration_seconds": (api_result or {}).get("duration_seconds"), } diff --git a/entry_points/run_pipeline.py b/entry_points/run_pipeline.py index 1334021..a5e33b5 100644 --- a/entry_points/run_pipeline.py +++ b/entry_points/run_pipeline.py @@ -18,7 +18,6 @@ import json import os import sys -import time from datetime import datetime, timezone from pathlib import Path @@ -31,6 +30,11 @@ from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec from processing_scripts.llm.slicers import get_slicer, is_empty_slice from processing_scripts.llm.templates import fill_template +from processing_scripts.llm_client.audit import ( + AuditRequestClient as PipelineClient, + AuditRequestConfig, + resolved_request_metadata, +) from processing_scripts.llm_preprocessing.semantic_checklist_01 import ( extract as cl01_extract, ) @@ -103,130 +107,6 @@ def estimate_tokens(text: str) -> int: return max(1, len(text) // 4) -class PipelineClient: - """Thin wrapper around the Anthropic, OpenAI, or Gemini API. - - Detects the provider from the model ID and uses the appropriate SDK. - Returns the same dict shape regardless of provider so downstream code - (``generate_report.py``, etc.) works unchanged. - """ - - def __init__(self, api_key: str, model: str, max_tokens: int = 8192): - from processing_scripts.llm_client.client import is_openai_model, is_gemini_model - - self.model = model - self.max_tokens = max_tokens - self._is_openai = is_openai_model(model) - self._is_gemini = is_gemini_model(model) - - if self._is_openai: - import openai - self._client = openai.OpenAI(api_key=api_key) - elif self._is_gemini: - from google import genai - self._client = genai.Client(api_key=api_key) - else: - import anthropic - self._client = anthropic.Anthropic(api_key=api_key) - - def call(self, prompt: str) -> dict: - """Send *prompt* to the API and return a result dict. - - Returns a dict with keys ``success``, ``response``, ``model``, - ``usage``, ``stop_reason``, and ``duration_seconds``. - """ - start = time.time() - - try: - if self._is_openai: - return self._call_openai(prompt, start) - if self._is_gemini: - return self._call_gemini(prompt, start) - return self._call_anthropic(prompt, start) - except Exception as e: - return { - "success": False, - "error": str(e), - "duration_seconds": round(time.time() - start, 2), - } - - def _call_anthropic(self, prompt: str, start: float) -> dict: - """Call the Anthropic Messages API.""" - from processing_scripts.llm_client.client import supports_temperature - - request_kwargs = { - "model": self.model, - "max_tokens": self.max_tokens, - "messages": [{"role": "user", "content": prompt}], - } - if supports_temperature(self.model): - request_kwargs["temperature"] = 0.1 - - message = self._client.messages.create(**request_kwargs) - - response_text = "".join( - block.text for block in message.content if block.type == "text" - ) - - return { - "success": True, - "response": response_text, - "model": message.model, - "usage": { - "input_tokens": message.usage.input_tokens, - "output_tokens": message.usage.output_tokens, - }, - "stop_reason": message.stop_reason, - "duration_seconds": round(time.time() - start, 2), - } - - def _call_openai(self, prompt: str, start: float) -> dict: - """Call the OpenAI Chat Completions API.""" - response = self._client.chat.completions.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=0.1, - messages=[{"role": "user", "content": prompt}], - ) - - return { - "success": True, - "response": response.choices[0].message.content, - "model": response.model, - "usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - }, - "stop_reason": response.choices[0].finish_reason, - "duration_seconds": round(time.time() - start, 2), - } - - def _call_gemini(self, prompt: str, start: float) -> dict: - """Call the Gemini generateContent API.""" - response = self._client.models.generate_content( - model=self.model, - contents=prompt, - config={ - "temperature": 0.1, - "max_output_tokens": self.max_tokens, - }, - ) - - return { - "success": True, - "response": response.text, - "model": self.model, - "usage": { - "input_tokens": response.usage_metadata.prompt_token_count, - "output_tokens": response.usage_metadata.candidates_token_count, - }, - "stop_reason": response.candidates[0].finish_reason.name - if response.candidates - else None, - "duration_seconds": round(time.time() - start, 2), - } - - def save_json(obj: object, path: Path) -> None: """Write an object as formatted JSON to a file.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -241,11 +121,17 @@ def run_pipeline( dry_run: bool, include_summaries: bool, progress_callback=None, + request_config: AuditRequestConfig | None = None, ) -> dict: """Execute the full element-specific accessibility audit pipeline. Returns a summary dict with run metadata and per-prompt results. """ + if request_config is None: + request_config = AuditRequestConfig(model=model) + elif request_config.model != model: + raise ValueError("model and request_config.model must match") + output_dir.mkdir(parents=True, exist_ok=True) html_path_str = str(html_path) @@ -327,7 +213,10 @@ def run_pipeline( # Initialise client once (only needed for live runs) client = None if not dry_run: - client = PipelineClient(api_key=api_key, model=model) + client = PipelineClient( + api_key=api_key, + request_config=request_config, + ) for spec in PROMPT_REGISTRY: # Skip summaries unless requested @@ -436,6 +325,7 @@ def run_pipeline( "run_timestamp": datetime.now(timezone.utc).isoformat(), "html_file": str(html_path), "model": model, + "request_config": resolved_request_metadata(request_config), "dry_run": dry_run, "include_summaries": include_summaries, "programmatic_findings_count": len(programmatic_findings), diff --git a/processing_scripts/llm_client/__init__.py b/processing_scripts/llm_client/__init__.py index 01e8aa9..7fa4c70 100644 --- a/processing_scripts/llm_client/__init__.py +++ b/processing_scripts/llm_client/__init__.py @@ -12,12 +12,15 @@ report = run_all(client, prompts, all_slices) """ +from .audit import AuditRequestClient, AuditRequestConfig from .client import AuditClient, OpenAIAuditClient, create_audit_client, is_openai_model from .prompt_loader import load_prompts, load_all_prompts from .runner import run_checklist, run_all __all__ = [ "AuditClient", + "AuditRequestClient", + "AuditRequestConfig", "OpenAIAuditClient", "create_audit_client", "is_openai_model", diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py new file mode 100644 index 0000000..472a672 --- /dev/null +++ b/processing_scripts/llm_client/audit.py @@ -0,0 +1,347 @@ +"""Provider-independent request configuration for accessibility audits.""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from .client import is_gemini_model, is_openai_model, supports_temperature + + +@dataclass(frozen=True) +class AuditRequestConfig: + """Configuration shared by every prompt in one audit run. + + ``temperature=None`` deliberately omits the sampling parameter. Existing + production callers retain the historical 0.1 temperature and 8,192-token + output limit when they do not supply this configuration explicitly. + """ + + model: str + reasoning_effort: str | None = None + temperature: float | None = 0.1 + max_output_tokens: int = 8192 + + def __post_init__(self) -> None: + if not self.model: + raise ValueError("model must not be empty") + if self.max_output_tokens <= 0: + raise ValueError("max_output_tokens must be greater than zero") + + def as_metadata(self) -> dict[str, Any]: + """Return the requested configuration as serializable metadata.""" + omitted = [] if self.temperature is not None else ["temperature"] + return { + "model": self.model, + "reasoning_effort": self.reasoning_effort, + "max_output_tokens": self.max_output_tokens, + "sampling": { + "temperature": self.temperature, + "omitted_parameters": omitted, + }, + } + + +def resolved_request_metadata(config: AuditRequestConfig) -> dict[str, Any]: + """Describe the effective request after provider capability omissions.""" + metadata = config.as_metadata() + if ( + not is_openai_model(config.model) + and not is_gemini_model(config.model) + and not supports_temperature(config.model) + ): + metadata["sampling"] = { + "temperature": None, + "omitted_parameters": ["temperature"], + } + return metadata + + +def _provider_data(value: Any) -> Any: + """Convert SDK response metadata into JSON-serializable builtins.""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Enum): + return value.value + if isinstance(value, Mapping): + return {str(key): _provider_data(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_provider_data(item) for item in value] + if hasattr(value, "model_dump"): + return _provider_data(value.model_dump()) + if hasattr(value, "to_dict"): + return _provider_data(value.to_dict()) + if hasattr(value, "__dict__"): + return { + key: _provider_data(item) + for key, item in vars(value).items() + if not key.startswith("_") + } + return str(value) + + +def _usage_result( + provider_usage: Any, + *, + input_tokens: int, + output_tokens: int, + total_tokens: int | None = None, + cached_input_tokens: int = 0, + cache_creation_input_tokens: int = 0, + reasoning_tokens: int = 0, +) -> dict[str, Any]: + """Return common token categories plus the lossless provider payload.""" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": ( + total_tokens + if total_tokens is not None + else input_tokens + output_tokens + ), + "cached_input_tokens": cached_input_tokens, + "cache_creation_input_tokens": cache_creation_input_tokens, + "reasoning_tokens": reasoning_tokens, + "provider_usage": _provider_data(provider_usage), + } + + +class AuditRequestClient: + """Send an audit prompt through Anthropic, OpenAI, or Gemini. + + The returned dict retains the legacy keys consumed by the production + pipeline and adds complete provider, request, usage, timing, stop, and + failure metadata for evaluation runs. + """ + + def __init__( + self, + api_key: str, + model: str | None = None, + max_tokens: int = 8192, + *, + request_config: AuditRequestConfig | None = None, + provider_client: Any | None = None, + ): + if request_config is None: + if model is None: + raise TypeError("model or request_config is required") + request_config = AuditRequestConfig( + model=model, + max_output_tokens=max_tokens, + ) + elif model is not None and model != request_config.model: + raise ValueError("model and request_config.model must match") + + self.request_config = request_config + self.model = request_config.model + self.max_tokens = request_config.max_output_tokens + self._is_openai = is_openai_model(self.model) + self._is_gemini = is_gemini_model(self.model) + + if request_config.reasoning_effort is not None and not self._is_openai: + raise ValueError( + "reasoning_effort is only supported for OpenAI audit requests" + ) + + if provider_client is not None: + self._client = provider_client + elif self._is_openai: + import openai + + self._client = openai.OpenAI(api_key=api_key) + elif self._is_gemini: + from google import genai + + self._client = genai.Client(api_key=api_key) + else: + import anthropic + + self._client = anthropic.Anthropic(api_key=api_key) + + def call(self, prompt: str) -> dict[str, Any]: + """Send one prompt and return a provider-neutral audit result.""" + start = time.monotonic() + + try: + if self._is_openai: + result = self._call_openai(prompt, start) + elif self._is_gemini: + result = self._call_gemini(prompt, start) + else: + result = self._call_anthropic(prompt, start) + result["request"] = resolved_request_metadata(self.request_config) + return result + except Exception as exc: + return { + "success": False, + "response": None, + "model": self.model, + "provider": self._provider_identity(), + "request": resolved_request_metadata(self.request_config), + "usage": _usage_result( + None, + input_tokens=0, + output_tokens=0, + ), + "stop_reason": None, + "error": str(exc), + "duration_seconds": round(time.monotonic() - start, 2), + "failure": { + "type": type(exc).__name__, + "message": str(exc), + "code": _provider_data(getattr(exc, "code", None)), + "status_code": getattr(exc, "status_code", None), + "request_id": getattr(exc, "request_id", None), + }, + } + + def _provider_identity(self) -> dict[str, Any]: + if self._is_openai: + return {"name": "openai", "endpoint": "chat.completions"} + if self._is_gemini: + return {"name": "google", "endpoint": "models.generate_content"} + return {"name": "anthropic", "endpoint": "messages"} + + def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: + request_kwargs = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + if ( + self.request_config.temperature is not None + and supports_temperature(self.model) + ): + request_kwargs["temperature"] = self.request_config.temperature + + message = self._client.messages.create(**request_kwargs) + response_text = "".join( + block.text for block in message.content if block.type == "text" + ) + + return { + "success": True, + "response": response_text, + "model": message.model, + "provider": { + "name": "anthropic", + "endpoint": "messages", + "response_id": getattr(message, "id", None), + "response_model": message.model, + "response_type": getattr(message, "type", None), + }, + "usage": _usage_result( + message.usage, + input_tokens=message.usage.input_tokens, + output_tokens=message.usage.output_tokens, + cached_input_tokens=getattr( + message.usage, "cache_read_input_tokens", 0 + ) + or 0, + cache_creation_input_tokens=getattr( + message.usage, "cache_creation_input_tokens", 0 + ) + or 0, + ), + "stop_reason": message.stop_reason, + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } + + def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: + request_kwargs = { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + } + if self.request_config.reasoning_effort is None: + request_kwargs["max_tokens"] = self.max_tokens + else: + request_kwargs["max_completion_tokens"] = self.max_tokens + request_kwargs["reasoning_effort"] = self.request_config.reasoning_effort + if self.request_config.temperature is not None: + request_kwargs["temperature"] = self.request_config.temperature + + response = self._client.chat.completions.create(**request_kwargs) + prompt_details = getattr(response.usage, "prompt_tokens_details", None) + completion_details = getattr( + response.usage, "completion_tokens_details", None + ) + + return { + "success": True, + "response": response.choices[0].message.content, + "model": response.model, + "provider": { + "name": "openai", + "endpoint": "chat.completions", + "response_id": getattr(response, "id", None), + "response_model": response.model, + "service_tier": getattr(response, "service_tier", None), + "system_fingerprint": getattr( + response, "system_fingerprint", None + ), + }, + "usage": _usage_result( + response.usage, + input_tokens=response.usage.prompt_tokens, + output_tokens=response.usage.completion_tokens, + total_tokens=getattr(response.usage, "total_tokens", None), + cached_input_tokens=getattr( + prompt_details, "cached_tokens", 0 + ) + or 0, + reasoning_tokens=getattr( + completion_details, "reasoning_tokens", 0 + ) + or 0, + ), + "stop_reason": response.choices[0].finish_reason, + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } + + def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: + generation_config = {"max_output_tokens": self.max_tokens} + if self.request_config.temperature is not None: + generation_config["temperature"] = self.request_config.temperature + + response = self._client.models.generate_content( + model=self.model, + contents=prompt, + config=generation_config, + ) + usage = response.usage_metadata + + return { + "success": True, + "response": response.text, + "model": self.model, + "provider": { + "name": "google", + "endpoint": "models.generate_content", + "response_id": getattr(response, "response_id", None), + "response_model": getattr(response, "model_version", self.model), + }, + "usage": _usage_result( + usage, + input_tokens=usage.prompt_token_count, + output_tokens=usage.candidates_token_count, + total_tokens=getattr(usage, "total_token_count", None), + cached_input_tokens=getattr( + usage, "cached_content_token_count", 0 + ) + or 0, + reasoning_tokens=getattr(usage, "thoughts_token_count", 0) or 0, + ), + "stop_reason": ( + response.candidates[0].finish_reason.name + if response.candidates + else None + ), + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py new file mode 100644 index 0000000..09a46ab --- /dev/null +++ b/tests/test_audit_request.py @@ -0,0 +1,336 @@ +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest import TestCase + +from entry_points.run_pipeline import ( + AuditRequestConfig, + PipelineClient, + run_pipeline, +) + + +class RecordingCompletions: + def __init__(self, response): + self.response = response + self.requests = [] + + def create(self, **kwargs): + self.requests.append(kwargs) + return self.response + + +class OpenAIFake: + def __init__(self, response): + self.chat = SimpleNamespace(completions=RecordingCompletions(response)) + + +class RecordingMessages: + def __init__(self, response): + self.response = response + self.requests = [] + + def create(self, **kwargs): + self.requests.append(kwargs) + return self.response + + +class AnthropicFake: + def __init__(self, response): + self.messages = RecordingMessages(response) + + +class RecordingModels: + def __init__(self, response): + self.response = response + self.requests = [] + + def generate_content(self, **kwargs): + self.requests.append(kwargs) + return self.response + + +class GeminiFake: + def __init__(self, response): + self.models = RecordingModels(response) + + +class AuditRequestTests(TestCase): + def test_luna_request_uses_medium_reasoning_without_temperature(self): + response = SimpleNamespace( + id="chatcmpl-eval-1", + model="gpt-5.6-luna", + service_tier="default", + system_fingerprint="fp_test", + choices=[ + SimpleNamespace( + message=SimpleNamespace(content='{"findings": []}'), + finish_reason="stop", + ) + ], + usage=SimpleNamespace( + prompt_tokens=120, + completion_tokens=30, + total_tokens=150, + prompt_tokens_details=SimpleNamespace(cached_tokens=20), + completion_tokens_details=SimpleNamespace(reasoning_tokens=10), + ), + ) + fake = OpenAIFake(response) + config = AuditRequestConfig( + model="gpt-5.6-luna", + reasoning_effort="medium", + temperature=None, + max_output_tokens=8192, + ) + + client = PipelineClient( + api_key="unused-by-fake", + request_config=config, + provider_client=fake, + ) + result = client.call("audit this payload") + + self.assertEqual( + fake.chat.completions.requests, + [ + { + "model": "gpt-5.6-luna", + "max_completion_tokens": 8192, + "reasoning_effort": "medium", + "messages": [ + {"role": "user", "content": "audit this payload"} + ], + } + ], + ) + self.assertTrue(result["success"]) + self.assertEqual( + result["provider"], + { + "name": "openai", + "endpoint": "chat.completions", + "response_id": "chatcmpl-eval-1", + "response_model": "gpt-5.6-luna", + "service_tier": "default", + "system_fingerprint": "fp_test", + }, + ) + self.assertEqual(result["usage"]["input_tokens"], 120) + self.assertEqual(result["usage"]["output_tokens"], 30) + self.assertEqual(result["usage"]["total_tokens"], 150) + self.assertEqual(result["usage"]["cached_input_tokens"], 20) + self.assertEqual(result["usage"]["reasoning_tokens"], 10) + self.assertEqual( + result["usage"]["provider_usage"]["prompt_tokens_details"], + {"cached_tokens": 20}, + ) + self.assertEqual(result["stop_reason"], "stop") + self.assertIsInstance(result["duration_seconds"], float) + self.assertIsNone(result["failure"]) + self.assertEqual( + result["request"], + { + "model": "gpt-5.6-luna", + "reasoning_effort": "medium", + "max_output_tokens": 8192, + "sampling": { + "temperature": None, + "omitted_parameters": ["temperature"], + }, + }, + ) + + def test_legacy_provider_request_defaults_are_preserved(self): + anthropic_response = SimpleNamespace( + id="msg_prod_1", + model="claude-sonnet-5", + type="message", + content=[SimpleNamespace(type="text", text="[]")], + usage=SimpleNamespace( + input_tokens=40, + output_tokens=12, + cache_creation_input_tokens=3, + cache_read_input_tokens=7, + ), + stop_reason="end_turn", + ) + anthropic_fake = AnthropicFake(anthropic_response) + + anthropic_result = PipelineClient( + "unused", "claude-sonnet-5", provider_client=anthropic_fake + ).call("audit") + + self.assertEqual( + anthropic_fake.messages.requests, + [ + { + "model": "claude-sonnet-5", + "max_tokens": 8192, + "messages": [{"role": "user", "content": "audit"}], + } + ], + ) + self.assertEqual(anthropic_result["provider"]["name"], "anthropic") + self.assertEqual(anthropic_result["usage"]["cached_input_tokens"], 7) + self.assertEqual( + anthropic_result["usage"]["cache_creation_input_tokens"], 3 + ) + self.assertEqual( + anthropic_result["request"]["sampling"], + { + "temperature": None, + "omitted_parameters": ["temperature"], + }, + ) + + openai_response = SimpleNamespace( + id="chatcmpl-prod-1", + model="gpt-4o", + choices=[ + SimpleNamespace( + message=SimpleNamespace(content="[]"), + finish_reason="stop", + ) + ], + usage=SimpleNamespace( + prompt_tokens=25, + completion_tokens=5, + total_tokens=30, + prompt_tokens_details=None, + completion_tokens_details=None, + ), + ) + openai_fake = OpenAIFake(openai_response) + + PipelineClient("unused", "gpt-4o", provider_client=openai_fake).call( + "audit" + ) + + self.assertEqual( + openai_fake.chat.completions.requests, + [ + { + "model": "gpt-4o", + "max_tokens": 8192, + "temperature": 0.1, + "messages": [{"role": "user", "content": "audit"}], + } + ], + ) + + gemini_response = SimpleNamespace( + response_id="gemini-prod-1", + model_version="gemini-flash-latest", + text="[]", + usage_metadata=SimpleNamespace( + prompt_token_count=31, + candidates_token_count=6, + total_token_count=39, + cached_content_token_count=2, + thoughts_token_count=2, + ), + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP") + ) + ], + ) + gemini_fake = GeminiFake(gemini_response) + + gemini_result = PipelineClient( + "unused", "gemini-flash-latest", provider_client=gemini_fake + ).call("audit") + + self.assertEqual( + gemini_fake.models.requests, + [ + { + "model": "gemini-flash-latest", + "contents": "audit", + "config": { + "temperature": 0.1, + "max_output_tokens": 8192, + }, + } + ], + ) + self.assertEqual(gemini_result["provider"]["name"], "google") + self.assertEqual(gemini_result["usage"]["cached_input_tokens"], 2) + self.assertEqual(gemini_result["usage"]["reasoning_tokens"], 2) + self.assertEqual(gemini_result["usage"]["total_tokens"], 39) + + def test_failure_exposes_request_and_provider_information(self): + class ProviderFailure(Exception): + status_code = 429 + code = "rate_limit_exceeded" + request_id = "req_failed_1" + + fake = OpenAIFake(None) + + def fail(**kwargs): + raise ProviderFailure("try again later") + + fake.chat.completions.create = fail + config = AuditRequestConfig( + model="gpt-5.6-luna", + reasoning_effort="medium", + temperature=None, + ) + + result = PipelineClient( + "unused", request_config=config, provider_client=fake + ).call("audit") + + self.assertFalse(result["success"]) + self.assertEqual(result["error"], "try again later") + self.assertEqual(result["provider"]["name"], "openai") + self.assertEqual(result["provider"]["endpoint"], "chat.completions") + self.assertEqual(result["model"], "gpt-5.6-luna") + self.assertEqual(result["usage"]["total_tokens"], 0) + self.assertIsNone(result["stop_reason"]) + self.assertEqual( + result["failure"], + { + "type": "ProviderFailure", + "message": "try again later", + "code": "rate_limit_exceeded", + "status_code": 429, + "request_id": "req_failed_1", + }, + ) + self.assertEqual(result["request"], config.as_metadata()) + + def test_pipeline_accepts_and_persists_explicit_request_configuration(self): + config = AuditRequestConfig( + model="gpt-5.6-luna", + reasoning_effort="medium", + temperature=None, + max_output_tokens=8192, + ) + + with TemporaryDirectory() as directory, redirect_stdout(StringIO()): + root = Path(directory) + html_path = root / "benchmark.html" + html_path.write_text( + "Benchmark" + "

Benchmark

", + encoding="utf-8", + ) + + manifest = run_pipeline( + html_path=str(html_path), + output_dir=root / "output", + api_key=None, + model="gpt-5.6-luna", + dry_run=True, + include_summaries=False, + request_config=config, + ) + + self.assertTrue(manifest["dry_run"]) + self.assertEqual(manifest["request_config"], config.as_metadata()) + self.assertEqual(manifest["total_input_tokens"], 0) + self.assertEqual(manifest["total_output_tokens"], 0) From c395efaae247242004953cbf503ca07c7dc2a85d Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 14:18:29 +0000 Subject: [PATCH 007/126] style(audit): address review findings --- processing_scripts/llm_client/audit.py | 32 ++++++++++++++++++++++---- tests/test_audit_request.py | 16 ++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index 472a672..95a4a8d 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -6,10 +6,20 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, Literal from .client import is_gemini_model, is_openai_model, supports_temperature +ReasoningEffort = Literal[ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", +] + @dataclass(frozen=True) class AuditRequestConfig: @@ -21,11 +31,12 @@ class AuditRequestConfig: """ model: str - reasoning_effort: str | None = None + reasoning_effort: ReasoningEffort | None = None temperature: float | None = 0.1 max_output_tokens: int = 8192 def __post_init__(self) -> None: + """Reject configuration values that cannot form an audit request.""" if not self.model: raise ValueError("model must not be empty") if self.max_output_tokens <= 0: @@ -126,6 +137,7 @@ def __init__( request_config: AuditRequestConfig | None = None, provider_client: Any | None = None, ): + """Resolve legacy arguments and initialize the selected SDK client.""" if request_config is None: if model is None: raise TypeError("model or request_config is required") @@ -200,6 +212,7 @@ def call(self, prompt: str) -> dict[str, Any]: } def _provider_identity(self) -> dict[str, Any]: + """Return stable provider and endpoint names for failed requests.""" if self._is_openai: return {"name": "openai", "endpoint": "chat.completions"} if self._is_gemini: @@ -207,6 +220,7 @@ def _provider_identity(self) -> dict[str, Any]: return {"name": "anthropic", "endpoint": "messages"} def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through Anthropic Messages.""" request_kwargs = { "model": self.model, "max_tokens": self.max_tokens, @@ -253,6 +267,7 @@ def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: } def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through OpenAI Chat Completions.""" request_kwargs = { "model": self.model, "messages": [{"role": "user", "content": prompt}], @@ -261,7 +276,9 @@ def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: request_kwargs["max_tokens"] = self.max_tokens else: request_kwargs["max_completion_tokens"] = self.max_tokens - request_kwargs["reasoning_effort"] = self.request_config.reasoning_effort + request_kwargs["reasoning_effort"] = ( + self.request_config.reasoning_effort + ) if self.request_config.temperature is not None: request_kwargs["temperature"] = self.request_config.temperature @@ -305,6 +322,7 @@ def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: } def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through Gemini generateContent.""" generation_config = {"max_output_tokens": self.max_tokens} if self.request_config.temperature is not None: generation_config["temperature"] = self.request_config.temperature @@ -324,7 +342,9 @@ def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: "name": "google", "endpoint": "models.generate_content", "response_id": getattr(response, "response_id", None), - "response_model": getattr(response, "model_version", self.model), + "response_model": getattr( + response, "model_version", self.model + ), }, "usage": _usage_result( usage, @@ -335,7 +355,9 @@ def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: usage, "cached_content_token_count", 0 ) or 0, - reasoning_tokens=getattr(usage, "thoughts_token_count", 0) or 0, + reasoning_tokens=( + getattr(usage, "thoughts_token_count", 0) or 0 + ), ), "stop_reason": ( response.candidates[0].finish_reason.name diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index 09a46ab..15faaf8 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -14,51 +14,61 @@ class RecordingCompletions: def __init__(self, response): + """Store the response returned by every fake completion.""" self.response = response self.requests = [] def create(self, **kwargs): + """Record request arguments and return the configured response.""" self.requests.append(kwargs) return self.response class OpenAIFake: def __init__(self, response): + """Expose the Chat Completions shape used by the OpenAI SDK.""" self.chat = SimpleNamespace(completions=RecordingCompletions(response)) class RecordingMessages: def __init__(self, response): + """Store the response returned by every fake message request.""" self.response = response self.requests = [] def create(self, **kwargs): + """Record request arguments and return the configured response.""" self.requests.append(kwargs) return self.response class AnthropicFake: def __init__(self, response): + """Expose the Messages shape used by the Anthropic SDK.""" self.messages = RecordingMessages(response) class RecordingModels: def __init__(self, response): + """Store the response returned by every fake content request.""" self.response = response self.requests = [] def generate_content(self, **kwargs): + """Record request arguments and return the configured response.""" self.requests.append(kwargs) return self.response class GeminiFake: def __init__(self, response): + """Expose the Models shape used by the Google Gen AI SDK.""" self.models = RecordingModels(response) class AuditRequestTests(TestCase): def test_luna_request_uses_medium_reasoning_without_temperature(self): + """Luna uses the explicit evaluation request shape.""" response = SimpleNamespace( id="chatcmpl-eval-1", model="gpt-5.6-luna", @@ -144,6 +154,7 @@ def test_luna_request_uses_medium_reasoning_without_temperature(self): ) def test_legacy_provider_request_defaults_are_preserved(self): + """Implicit configuration preserves all production request defaults.""" anthropic_response = SimpleNamespace( id="msg_prod_1", model="claude-sonnet-5", @@ -263,6 +274,7 @@ def test_legacy_provider_request_defaults_are_preserved(self): self.assertEqual(gemini_result["usage"]["total_tokens"], 39) def test_failure_exposes_request_and_provider_information(self): + """Failures retain the metadata needed for later classification.""" class ProviderFailure(Exception): status_code = 429 code = "rate_limit_exceeded" @@ -271,6 +283,7 @@ class ProviderFailure(Exception): fake = OpenAIFake(None) def fail(**kwargs): + """Raise a provider-shaped failure without a network request.""" raise ProviderFailure("try again later") fake.chat.completions.create = fail @@ -303,7 +316,8 @@ def fail(**kwargs): ) self.assertEqual(result["request"], config.as_metadata()) - def test_pipeline_accepts_and_persists_explicit_request_configuration(self): + def test_pipeline_persists_explicit_request_configuration(self): + """The pipeline records explicit configuration without going live.""" config = AuditRequestConfig( model="gpt-5.6-luna", reasoning_effort="medium", From aa77ee9f7acde85f2c2e3ab5f1cdfb7a609cff40 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 14:31:41 +0000 Subject: [PATCH 008/126] feat(evaluation): bootstrap private CLI workspace --- .gitignore | 4 + README.md | 21 +++++ entry_points/model_evaluation.py | 6 ++ pyproject.toml | 14 +++- requirements.txt | 4 + tests/test_evaluation_cli.py | 111 +++++++++++++++++++++++++ uv.lock | 83 +++++++++++++++++++ vision_aid/evaluation/__init__.py | 1 + vision_aid/evaluation/cli.py | 128 +++++++++++++++++++++++++++++ vision_aid/evaluation/workspace.py | 57 +++++++++++++ 10 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 entry_points/model_evaluation.py create mode 100644 tests/test_evaluation_cli.py create mode 100644 vision_aid/evaluation/__init__.py create mode 100644 vision_aid/evaluation/cli.py create mode 100644 vision_aid/evaluation/workspace.py diff --git a/.gitignore b/.gitignore index 091970c..3784651 100644 --- a/.gitignore +++ b/.gitignore @@ -212,6 +212,10 @@ output/ # CI smoke-test output ci-output/ +# Private model-evaluation inputs and derived artifacts +/Pristine Accessibility Defect Report.xlsx +/.model-evaluation/ + # Node.js / Next.js generated files node_modules/ .next/ diff --git a/README.md b/README.md index 909446b..c5ce3cf 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,27 @@ GEMINI_API_KEY=AIza... > checks only, no LLM findings, no CSV, and still a `200 OK` from the web app. > The only signal is `summary.dry_run` in the response. +## Private Model Evaluation + +The evaluation CLI keeps the employer-provided workbook and every derived +artifact outside version control. Obtain +`Pristine Accessibility Defect Report.xlsx` from an authorized project source, +place it in the repository root, and initialize the private workspace: + +```bash +uv run visionaid-evaluate prepare +``` + +Use `--workbook PATH` for another authorized local copy and `--workspace PATH` +to override the default `.model-evaluation/` workspace. The workspace contains +separate `references/`, `reviews/`, `snapshots/`, `runs/`, and `reports/` +partitions, all covered by one ignore rule. + +Run `uv run visionaid-evaluate --help` to discover the `prepare`, `audit`, +`review`, and `report` workflows. At this bootstrap milestone, `audit` is +always a network-free dry run—even when an API key is present—and live +execution is not exposed. + ## Running the Web App ```bash diff --git a/entry_points/model_evaluation.py b/entry_points/model_evaluation.py new file mode 100644 index 0000000..9fd98b0 --- /dev/null +++ b/entry_points/model_evaluation.py @@ -0,0 +1,6 @@ +"""Repository entry point for private model-evaluation workflows.""" + +from vision_aid.evaluation.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 29ab6af..f99dfb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,21 @@ dependencies = [ "lxml>=4.9.0", "openai>=1.0.0", "google-genai>=1.0.0", + "openpyxl>=3.1.5", "python-dotenv~=1.2.1", "requests>=2.32.2", ] +[project.scripts] +visionaid-evaluate = "vision_aid.evaluation.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8.3.5", +] + +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.setuptools] -packages = ["vision_aid"] \ No newline at end of file +packages = ["vision_aid", "vision_aid.evaluation"] diff --git a/requirements.txt b/requirements.txt index 1a5da01..d70272b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,8 @@ distro==1.9.0 # openai docstring-parser==0.18.0 # via anthropic +et-xmlfile==2.0.0 + # via openpyxl google-auth==2.56.3 # via google-genai google-genai==2.17.0 @@ -58,6 +60,8 @@ lxml==6.1.1 # via visionaid openai==2.53.0 # via visionaid +openpyxl==3.1.5 + # via visionaid pyasn1==0.6.4 # via pyasn1-modules pyasn1-modules==0.4.2 diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py new file mode 100644 index 0000000..f5d4712 --- /dev/null +++ b/tests/test_evaluation_cli.py @@ -0,0 +1,111 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).parents[1] + + +def run_cli( + *arguments: str, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + """Run the installed evaluation module through its public CLI boundary.""" + command_env = os.environ.copy() + if env: + command_env.update(env) + return subprocess.run( + [sys.executable, "-m", "vision_aid.evaluation.cli", *arguments], + cwd=PROJECT_ROOT, + env=command_env, + capture_output=True, + text=True, + check=False, + ) + + +def test_help_exposes_the_evaluation_workflows() -> None: + """The CLI makes every evaluation lifecycle workflow discoverable.""" + result = run_cli("--help") + + assert result.returncode == 0 + for workflow in ("prepare", "audit", "review", "report"): + assert workflow in result.stdout + + +def test_prepare_initializes_one_partitioned_private_workspace(tmp_path: Path) -> None: + """Prepare creates all private artifact partitions around a local workbook.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + workspace = tmp_path / "private-evaluation" + + result = run_cli( + "prepare", + "--workspace", + str(workspace), + "--workbook", + str(workbook), + ) + + assert result.returncode == 0, result.stderr + assert "Private evaluation workspace initialized" in result.stdout + assert str(workbook.resolve()) in result.stdout + assert sorted(path.name for path in workspace.iterdir()) == [ + "references", + "reports", + "reviews", + "runs", + "snapshots", + ] + + +def test_prepare_explains_how_to_supply_a_missing_private_workbook( + tmp_path: Path, +) -> None: + """A missing workbook fails with authorized, local recovery instructions.""" + missing_workbook = tmp_path / "missing.xlsx" + workspace = tmp_path / "private-evaluation" + + result = run_cli( + "prepare", + "--workspace", + str(workspace), + "--workbook", + str(missing_workbook), + ) + + assert result.returncode == 2 + assert "Private source workbook not found" in result.stderr + assert "authorized project source" in result.stderr + assert "--workbook PATH" in result.stderr + assert not workspace.exists() + + +@pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) +def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( + tmp_path: Path, api_key_name: str +) -> None: + """Environment credentials alone can never make evaluation billable.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + workspace = tmp_path / "private-evaluation" + prepared = run_cli( + "prepare", + "--workspace", + str(workspace), + "--workbook", + str(workbook), + ) + assert prepared.returncode == 0, prepared.stderr + + result = run_cli( + "audit", + "--workspace", + str(workspace), + env={api_key_name: "must-not-be-used"}, + ) + + assert result.returncode == 0, result.stderr + assert "DRY RUN" in result.stdout + assert "No network requests were made" in result.stdout diff --git a/uv.lock b/uv.lock index 4693dfe..ce76b0b 100644 --- a/uv.lock +++ b/uv.lock @@ -324,6 +324,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "google-auth" version = "2.56.3" @@ -409,6 +418,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "jiter" version = "0.16.0" @@ -616,6 +634,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pyasn1" version = "0.6.4" @@ -763,6 +811,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" @@ -866,10 +939,16 @@ dependencies = [ { name = "google-genai" }, { name = "lxml" }, { name = "openai" }, + { name = "openpyxl" }, { name = "python-dotenv" }, { name = "requests" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "anthropic", specifier = "~=0.84.0" }, @@ -877,10 +956,14 @@ requires-dist = [ { name = "google-genai", specifier = ">=1.0.0" }, { name = "lxml", specifier = ">=4.9.0" }, { name = "openai", specifier = ">=1.0.0" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "python-dotenv", specifier = "~=1.2.1" }, { name = "requests", specifier = ">=2.32.2" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3.5" }] + [[package]] name = "websockets" version = "16.1.1" diff --git a/vision_aid/evaluation/__init__.py b/vision_aid/evaluation/__init__.py new file mode 100644 index 0000000..c734d16 --- /dev/null +++ b/vision_aid/evaluation/__init__.py @@ -0,0 +1 @@ +"""Private model-evaluation workflows.""" diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py new file mode 100644 index 0000000..40f979e --- /dev/null +++ b/vision_aid/evaluation/cli.py @@ -0,0 +1,128 @@ +"""Command-line shell for the private model-evaluation lifecycle.""" + +import argparse +from collections.abc import Callable, Sequence +from pathlib import Path + +from vision_aid.evaluation.workspace import ( + DEFAULT_WORKBOOK_FILENAME, + DEFAULT_WORKSPACE, + PrivateInputError, + PrivateWorkspace, + require_private_workbook, +) + +CommandHandler = Callable[[argparse.Namespace], int] + + +def _add_workspace_argument(parser: argparse.ArgumentParser) -> None: + """Add the shared private-workspace option to a workflow parser.""" + parser.add_argument( + "--workspace", + type=Path, + default=DEFAULT_WORKSPACE, + help=f"private artifact workspace (default: {DEFAULT_WORKSPACE})", + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the discoverable evaluation command hierarchy.""" + parser = argparse.ArgumentParser( + prog="visionaid-evaluate", + description=( + "Prepare, audit, review, and report private model evaluations " + "without committing private data or spending money by default." + ), + ) + workflows = parser.add_subparsers( + title="workflows", + dest="workflow", + required=True, + ) + + prepare = workflows.add_parser( + "prepare", + help="initialize private inputs and artifact directories", + ) + _add_workspace_argument(prepare) + prepare.add_argument( + "--workbook", + type=Path, + default=Path(DEFAULT_WORKBOOK_FILENAME), + help=( + f"authorized local source workbook (default: ./{DEFAULT_WORKBOOK_FILENAME})" + ), + ) + prepare.set_defaults(handler=_prepare) + + audit = workflows.add_parser( + "audit", + help="inspect the prepared audit plan in network-free dry-run mode", + ) + _add_workspace_argument(audit) + audit.set_defaults(handler=_audit) + + review = workflows.add_parser( + "review", + help="work with private eligibility and match review artifacts", + ) + _add_workspace_argument(review) + review.set_defaults(handler=_review) + + report = workflows.add_parser( + "report", + help="generate private evaluation reports", + ) + _add_workspace_argument(report) + report.set_defaults(handler=_report) + return parser + + +def _prepare(arguments: argparse.Namespace) -> int: + """Validate the private workbook before creating the workspace layout.""" + workbook = require_private_workbook(arguments.workbook) + workspace = PrivateWorkspace(arguments.workspace) + workspace.initialize() + print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") + print(f"Private source workbook: {workbook}") + return 0 + + +def _audit(arguments: argparse.Namespace) -> int: + """Expose the safe audit shell without enabling live execution yet.""" + workspace = PrivateWorkspace(arguments.workspace) + workspace.require_initialized() + print("DRY RUN: No network requests were made and no API usage was incurred.") + print(f"Private evaluation workspace: {workspace.root.resolve()}") + return 0 + + +def _review(arguments: argparse.Namespace) -> int: + """Expose the private review workflow shell.""" + workspace = PrivateWorkspace(arguments.workspace) + workspace.require_initialized() + print(f"Private review workspace: {(workspace.root / 'reviews').resolve()}") + return 0 + + +def _report(arguments: argparse.Namespace) -> int: + """Expose the private report workflow shell.""" + workspace = PrivateWorkspace(arguments.workspace) + workspace.require_initialized() + print(f"Private report workspace: {(workspace.root / 'reports').resolve()}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the requested evaluation workflow and render actionable errors.""" + parser = build_parser() + arguments = parser.parse_args(argv) + handler: CommandHandler = arguments.handler + try: + return handler(arguments) + except PrivateInputError as error: + parser.error(str(error)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py new file mode 100644 index 0000000..c6d09e0 --- /dev/null +++ b/vision_aid/evaluation/workspace.py @@ -0,0 +1,57 @@ +"""Private filesystem conventions for model evaluation artifacts.""" + +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_WORKBOOK_FILENAME = "Pristine Accessibility Defect Report.xlsx" +DEFAULT_WORKSPACE = Path(".model-evaluation") +WORKSPACE_PARTITIONS = ( + "references", + "reviews", + "snapshots", + "runs", + "reports", +) + + +class PrivateInputError(ValueError): + """Report a missing local input without attempting to replace it.""" + + +@dataclass(frozen=True) +class PrivateWorkspace: + """Paths reserved for private evaluation inputs and derived artifacts.""" + + root: Path + + @property + def partitions(self) -> tuple[Path, ...]: + """Return every required private artifact partition.""" + return tuple(self.root / name for name in WORKSPACE_PARTITIONS) + + def initialize(self) -> None: + """Create the private workspace and all of its artifact partitions.""" + for partition in self.partitions: + partition.mkdir(parents=True, exist_ok=True) + + def require_initialized(self) -> None: + """Fail with recovery instructions unless every partition exists.""" + missing = [path.name for path in self.partitions if not path.is_dir()] + if missing: + raise PrivateInputError( + f"Private evaluation workspace is not initialized at " + f"{self.root.resolve()}. Run 'visionaid-evaluate prepare' " + "with an authorized local workbook first." + ) + + +def require_private_workbook(path: Path) -> Path: + """Resolve a local workbook or explain how an authorized user supplies it.""" + if not path.is_file(): + raise PrivateInputError( + f"Private source workbook not found at {path.resolve()}. Obtain " + f"'{DEFAULT_WORKBOOK_FILENAME}' from an authorized project source " + "and place it at the repository root, or pass --workbook PATH. " + "The evaluation CLI will not download or substitute private inputs." + ) + return path.resolve() From b99017f8dbc26a2ef3ae3d85cb09c3d567d744d7 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 14:35:52 +0000 Subject: [PATCH 009/126] fix(evaluation): enforce private workspace safety --- README.md | 8 ++--- tests/test_evaluation_cli.py | 52 ++++++++++++++++++++++-------- vision_aid/evaluation/cli.py | 22 +++---------- vision_aid/evaluation/workspace.py | 25 +++++++++++++- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index c5ce3cf..bfb239d 100644 --- a/README.md +++ b/README.md @@ -249,10 +249,10 @@ place it in the repository root, and initialize the private workspace: uv run visionaid-evaluate prepare ``` -Use `--workbook PATH` for another authorized local copy and `--workspace PATH` -to override the default `.model-evaluation/` workspace. The workspace contains -separate `references/`, `reviews/`, `snapshots/`, `runs/`, and `reports/` -partitions, all covered by one ignore rule. +Use `--workbook PATH` for another authorized local copy outside the repository. +The fixed `.model-evaluation/` workspace contains separate `references/`, +`reviews/`, `snapshots/`, `runs/`, and `reports/` partitions, all covered by one +ignore rule. Run `uv run visionaid-evaluate --help` to discover the `prepare`, `audit`, `review`, and `report` workflows. At this bootstrap milestone, `audit` is diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index f5d4712..686ec64 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -2,6 +2,7 @@ import subprocess import sys from pathlib import Path +from tempfile import TemporaryDirectory import pytest @@ -9,7 +10,9 @@ def run_cli( - *arguments: str, env: dict[str, str] | None = None + *arguments: str, + cwd: Path = PROJECT_ROOT, + env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Run the installed evaluation module through its public CLI boundary.""" command_env = os.environ.copy() @@ -17,7 +20,7 @@ def run_cli( command_env.update(env) return subprocess.run( [sys.executable, "-m", "vision_aid.evaluation.cli", *arguments], - cwd=PROJECT_ROOT, + cwd=cwd, env=command_env, capture_output=True, text=True, @@ -38,14 +41,13 @@ def test_prepare_initializes_one_partitioned_private_workspace(tmp_path: Path) - """Prepare creates all private artifact partitions around a local workbook.""" workbook = tmp_path / "synthetic-reference.xlsx" workbook.write_bytes(b"synthetic workbook placeholder") - workspace = tmp_path / "private-evaluation" + workspace = tmp_path / ".model-evaluation" result = run_cli( "prepare", - "--workspace", - str(workspace), "--workbook", str(workbook), + cwd=tmp_path, ) assert result.returncode == 0, result.stderr @@ -65,14 +67,13 @@ def test_prepare_explains_how_to_supply_a_missing_private_workbook( ) -> None: """A missing workbook fails with authorized, local recovery instructions.""" missing_workbook = tmp_path / "missing.xlsx" - workspace = tmp_path / "private-evaluation" + workspace = tmp_path / ".model-evaluation" result = run_cli( "prepare", - "--workspace", - str(workspace), "--workbook", str(missing_workbook), + cwd=tmp_path, ) assert result.returncode == 2 @@ -82,6 +83,19 @@ def test_prepare_explains_how_to_supply_a_missing_private_workbook( assert not workspace.exists() +def test_prepare_rejects_an_unignored_workbook_inside_the_worktree() -> None: + """A workbook override cannot make private data visible to version control.""" + with TemporaryDirectory(dir=PROJECT_ROOT) as temporary_directory: + workbook = Path(temporary_directory) / "private-input.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + + result = run_cli("prepare", "--workbook", str(workbook)) + + assert result.returncode == 2 + assert "would be visible to version control" in result.stderr + assert "outside the repository" in result.stderr + + @pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( tmp_path: Path, api_key_name: str @@ -89,21 +103,31 @@ def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( """Environment credentials alone can never make evaluation billable.""" workbook = tmp_path / "synthetic-reference.xlsx" workbook.write_bytes(b"synthetic workbook placeholder") - workspace = tmp_path / "private-evaluation" prepared = run_cli( "prepare", - "--workspace", - str(workspace), "--workbook", str(workbook), + cwd=tmp_path, ) assert prepared.returncode == 0, prepared.stderr + network_guard = tmp_path / "network_guard" + network_guard.mkdir() + (network_guard / "sitecustomize.py").write_text( + "import socket\n" + "def blocked_socket(*args, **kwargs):\n" + " raise AssertionError('evaluation CLI attempted network access')\n" + "socket.socket = blocked_socket\n", + encoding="utf-8", + ) + result = run_cli( "audit", - "--workspace", - str(workspace), - env={api_key_name: "must-not-be-used"}, + cwd=tmp_path, + env={ + api_key_name: "must-not-be-used", + "PYTHONPATH": str(network_guard), + }, ) assert result.returncode == 0, result.stderr diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 40f979e..b3606c0 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -15,16 +15,6 @@ CommandHandler = Callable[[argparse.Namespace], int] -def _add_workspace_argument(parser: argparse.ArgumentParser) -> None: - """Add the shared private-workspace option to a workflow parser.""" - parser.add_argument( - "--workspace", - type=Path, - default=DEFAULT_WORKSPACE, - help=f"private artifact workspace (default: {DEFAULT_WORKSPACE})", - ) - - def build_parser() -> argparse.ArgumentParser: """Build the discoverable evaluation command hierarchy.""" parser = argparse.ArgumentParser( @@ -44,7 +34,6 @@ def build_parser() -> argparse.ArgumentParser: "prepare", help="initialize private inputs and artifact directories", ) - _add_workspace_argument(prepare) prepare.add_argument( "--workbook", type=Path, @@ -59,21 +48,18 @@ def build_parser() -> argparse.ArgumentParser: "audit", help="inspect the prepared audit plan in network-free dry-run mode", ) - _add_workspace_argument(audit) audit.set_defaults(handler=_audit) review = workflows.add_parser( "review", help="work with private eligibility and match review artifacts", ) - _add_workspace_argument(review) review.set_defaults(handler=_review) report = workflows.add_parser( "report", help="generate private evaluation reports", ) - _add_workspace_argument(report) report.set_defaults(handler=_report) return parser @@ -81,7 +67,7 @@ def build_parser() -> argparse.ArgumentParser: def _prepare(arguments: argparse.Namespace) -> int: """Validate the private workbook before creating the workspace layout.""" workbook = require_private_workbook(arguments.workbook) - workspace = PrivateWorkspace(arguments.workspace) + workspace = PrivateWorkspace(DEFAULT_WORKSPACE) workspace.initialize() print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") print(f"Private source workbook: {workbook}") @@ -90,7 +76,7 @@ def _prepare(arguments: argparse.Namespace) -> int: def _audit(arguments: argparse.Namespace) -> int: """Expose the safe audit shell without enabling live execution yet.""" - workspace = PrivateWorkspace(arguments.workspace) + workspace = PrivateWorkspace(DEFAULT_WORKSPACE) workspace.require_initialized() print("DRY RUN: No network requests were made and no API usage was incurred.") print(f"Private evaluation workspace: {workspace.root.resolve()}") @@ -99,7 +85,7 @@ def _audit(arguments: argparse.Namespace) -> int: def _review(arguments: argparse.Namespace) -> int: """Expose the private review workflow shell.""" - workspace = PrivateWorkspace(arguments.workspace) + workspace = PrivateWorkspace(DEFAULT_WORKSPACE) workspace.require_initialized() print(f"Private review workspace: {(workspace.root / 'reviews').resolve()}") return 0 @@ -107,7 +93,7 @@ def _review(arguments: argparse.Namespace) -> int: def _report(arguments: argparse.Namespace) -> int: """Expose the private report workflow shell.""" - workspace = PrivateWorkspace(arguments.workspace) + workspace = PrivateWorkspace(DEFAULT_WORKSPACE) workspace.require_initialized() print(f"Private report workspace: {(workspace.root / 'reports').resolve()}") return 0 diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py index c6d09e0..48ea63a 100644 --- a/vision_aid/evaluation/workspace.py +++ b/vision_aid/evaluation/workspace.py @@ -45,6 +45,15 @@ def require_initialized(self) -> None: ) +def find_worktree_root(start: Path) -> Path | None: + """Find the containing Git worktree without invoking Git.""" + resolved_start = start.resolve() + for directory in (resolved_start, *resolved_start.parents): + if (directory / ".git").exists(): + return directory + return None + + def require_private_workbook(path: Path) -> Path: """Resolve a local workbook or explain how an authorized user supplies it.""" if not path.is_file(): @@ -54,4 +63,18 @@ def require_private_workbook(path: Path) -> Path: "and place it at the repository root, or pass --workbook PATH. " "The evaluation CLI will not download or substitute private inputs." ) - return path.resolve() + workbook = path.resolve() + worktree = find_worktree_root(Path.cwd()) + if worktree is None or not workbook.is_relative_to(worktree): + return workbook + + default_workbook = (worktree / DEFAULT_WORKBOOK_FILENAME).resolve() + private_workspace = (worktree / DEFAULT_WORKSPACE).resolve() + if workbook != default_workbook and not workbook.is_relative_to(private_workspace): + raise PrivateInputError( + f"Private source workbook at {workbook} would be visible to " + "version control. Keep the exact default filename at the repository " + "root, store the override inside .model-evaluation, or pass a path " + "outside the repository." + ) + return workbook From e9485bf4d867099db53e031718831f2d08b5e542 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 14:38:12 +0000 Subject: [PATCH 010/126] fix(evaluation): anchor private paths to worktree --- tests/test_evaluation_cli.py | 33 ++++++++++++++++++++++++++++-- vision_aid/evaluation/cli.py | 9 ++++---- vision_aid/evaluation/workspace.py | 20 ++++++++++++++---- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 686ec64..ac0d8bf 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -62,6 +62,28 @@ def test_prepare_initializes_one_partitioned_private_workspace(tmp_path: Path) - ] +def test_prepare_anchors_default_private_paths_to_the_worktree_root( + tmp_path: Path, +) -> None: + """Invocation from a subdirectory still uses the root-level ignore rules.""" + worktree = tmp_path / "worktree" + (worktree / ".git").mkdir(parents=True) + (worktree / ".gitignore").write_text( + "/Pristine Accessibility Defect Report.xlsx\n/.model-evaluation/\n", + encoding="utf-8", + ) + workbook = worktree / "Pristine Accessibility Defect Report.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + nested_directory = worktree / "nested" + nested_directory.mkdir() + + result = run_cli("prepare", cwd=nested_directory) + + assert result.returncode == 0, result.stderr + assert (worktree / ".model-evaluation").is_dir() + assert not (nested_directory / ".model-evaluation").exists() + + def test_prepare_explains_how_to_supply_a_missing_private_workbook( tmp_path: Path, ) -> None: @@ -83,13 +105,20 @@ def test_prepare_explains_how_to_supply_a_missing_private_workbook( assert not workspace.exists() -def test_prepare_rejects_an_unignored_workbook_inside_the_worktree() -> None: +def test_prepare_rejects_an_unignored_workbook_inside_the_worktree( + tmp_path: Path, +) -> None: """A workbook override cannot make private data visible to version control.""" with TemporaryDirectory(dir=PROJECT_ROOT) as temporary_directory: workbook = Path(temporary_directory) / "private-input.xlsx" workbook.write_bytes(b"synthetic workbook placeholder") - result = run_cli("prepare", "--workbook", str(workbook)) + result = run_cli( + "prepare", + "--workbook", + str(workbook), + cwd=tmp_path, + ) assert result.returncode == 2 assert "would be visible to version control" in result.stderr diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index b3606c0..b3a9f98 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -6,7 +6,6 @@ from vision_aid.evaluation.workspace import ( DEFAULT_WORKBOOK_FILENAME, - DEFAULT_WORKSPACE, PrivateInputError, PrivateWorkspace, require_private_workbook, @@ -67,7 +66,7 @@ def build_parser() -> argparse.ArgumentParser: def _prepare(arguments: argparse.Namespace) -> int: """Validate the private workbook before creating the workspace layout.""" workbook = require_private_workbook(arguments.workbook) - workspace = PrivateWorkspace(DEFAULT_WORKSPACE) + workspace = PrivateWorkspace.from_current_directory() workspace.initialize() print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") print(f"Private source workbook: {workbook}") @@ -76,7 +75,7 @@ def _prepare(arguments: argparse.Namespace) -> int: def _audit(arguments: argparse.Namespace) -> int: """Expose the safe audit shell without enabling live execution yet.""" - workspace = PrivateWorkspace(DEFAULT_WORKSPACE) + workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() print("DRY RUN: No network requests were made and no API usage was incurred.") print(f"Private evaluation workspace: {workspace.root.resolve()}") @@ -85,7 +84,7 @@ def _audit(arguments: argparse.Namespace) -> int: def _review(arguments: argparse.Namespace) -> int: """Expose the private review workflow shell.""" - workspace = PrivateWorkspace(DEFAULT_WORKSPACE) + workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() print(f"Private review workspace: {(workspace.root / 'reviews').resolve()}") return 0 @@ -93,7 +92,7 @@ def _review(arguments: argparse.Namespace) -> int: def _report(arguments: argparse.Namespace) -> int: """Expose the private report workflow shell.""" - workspace = PrivateWorkspace(DEFAULT_WORKSPACE) + workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() print(f"Private report workspace: {(workspace.root / 'reports').resolve()}") return 0 diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py index 48ea63a..2e01e66 100644 --- a/vision_aid/evaluation/workspace.py +++ b/vision_aid/evaluation/workspace.py @@ -24,6 +24,13 @@ class PrivateWorkspace: root: Path + @classmethod + def from_current_directory(cls) -> "PrivateWorkspace": + """Anchor artifacts at the worktree root or the non-versioned CWD.""" + current_directory = Path.cwd().resolve() + worktree = find_worktree_root(current_directory) + return cls((worktree or current_directory) / DEFAULT_WORKSPACE) + @property def partitions(self) -> tuple[Path, ...]: """Return every required private artifact partition.""" @@ -56,15 +63,20 @@ def find_worktree_root(start: Path) -> Path | None: def require_private_workbook(path: Path) -> Path: """Resolve a local workbook or explain how an authorized user supplies it.""" - if not path.is_file(): + candidate = path + current_worktree = find_worktree_root(Path.cwd()) + if path == Path(DEFAULT_WORKBOOK_FILENAME) and current_worktree is not None: + candidate = current_worktree / path + + if not candidate.is_file(): raise PrivateInputError( - f"Private source workbook not found at {path.resolve()}. Obtain " + f"Private source workbook not found at {candidate.resolve()}. Obtain " f"'{DEFAULT_WORKBOOK_FILENAME}' from an authorized project source " "and place it at the repository root, or pass --workbook PATH. " "The evaluation CLI will not download or substitute private inputs." ) - workbook = path.resolve() - worktree = find_worktree_root(Path.cwd()) + workbook = candidate.resolve() + worktree = find_worktree_root(workbook.parent) if worktree is None or not workbook.is_relative_to(worktree): return workbook From e3de4fa83c9a8dc05a19efe4981a3efe3b40ef36 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 10:44:25 -0400 Subject: [PATCH 011/126] docs(agents): define ticket resolution convention --- ...01-explicit-audit-request-configuration.md | 16 +++++++++----- .../issues/02-private-evaluation-workspace.md | 21 ++++++++++++------- docs/agents/issue-tracker.md | 13 ++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md index 2e404ac..a069642 100644 --- a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md +++ b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md @@ -4,9 +4,15 @@ **Blocked by:** None (can start immediately). -**Status:** ready-for-agent +**Status:** resolved -- [ ] Existing Anthropic, OpenAI, and Gemini production callers retain their current effective defaults. -- [ ] An OpenAI evaluation caller can explicitly request GPT-5.6 Luna with medium reasoning and no temperature parameter. -- [ ] The request result exposes the provider metadata, token categories, stop reason, duration, and failure information required by evaluation runs. -- [ ] Network-free tests verify legacy defaults and the Luna request shape through fakes. +- [x] Existing Anthropic, OpenAI, and Gemini production callers retain their current effective defaults. +- [x] An OpenAI evaluation caller can explicitly request GPT-5.6 Luna with medium reasoning and no temperature parameter. +- [x] The request result exposes the provider metadata, token categories, stop reason, duration, and failure information required by evaluation runs. +- [x] Network-free tests verify legacy defaults and the Luna request shape through fakes. + +## Comments + +- Implemented in `c371979` with review fixes in `c395efa`. Verified by four + network-free audit-request tests and the production pipeline dry-run smoke + test. diff --git a/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md b/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md index bc7bcca..8e14afc 100644 --- a/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md +++ b/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md @@ -4,11 +4,16 @@ **Blocked by:** None (can start immediately). -**Status:** ready-for-agent - -- [ ] The evaluation CLI exposes discoverable prepare, audit, review, and report workflows. -- [ ] The source workbook is ignored by exact filename, and all generated references, reviews, snapshots, runs, and reports are ignored under one private workspace. -- [ ] Missing private inputs produce actionable instructions rather than downloads or silent substitutions. -- [ ] Dry-run behavior is the default even when an API key is present. -- [ ] Excel support is added as a runtime dependency and pytest as a development dependency, with lock and generated dependency artifacts kept consistent. -- [ ] Synthetic CLI tests prove workspace initialization and safe defaults without private data or network access. +**Status:** resolved + +- [x] The evaluation CLI exposes discoverable prepare, audit, review, and report workflows. +- [x] The source workbook is ignored by exact filename, and all generated references, reviews, snapshots, runs, and reports are ignored under one private workspace. +- [x] Missing private inputs produce actionable instructions rather than downloads or silent substitutions. +- [x] Dry-run behavior is the default even when an API key is present. +- [x] Excel support is added as a runtime dependency and pytest as a development dependency, with lock and generated dependency artifacts kept consistent. +- [x] Synthetic CLI tests prove workspace initialization and safe defaults without private data or network access. + +## Comments + +- Implemented in `aa77ee9` with safety fixes in `b99017f` and `e9485bf`. + Verified by the synthetic evaluation CLI test suite. diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md index 0209a19..7ba4a25 100644 --- a/docs/agents/issue-tracker.md +++ b/docs/agents/issue-tracker.md @@ -10,6 +10,19 @@ Issues and specs for this repo live as markdown files in `.scratch/`. - Triage state is recorded as a `Status:` line near the top of each issue file (see `triage-labels.md` for the role strings) - Comments and conversation history append to the bottom of the file under a `## Comments` heading +## Completing implementation issues + +Close an implementation issue only after its implementation, verification, and +review fixes are committed: + +- Change each demonstrated acceptance criterion from `[ ]` to `[x]`. Leave the + issue open if any criterion is unmet. +- Change `Status:` to `resolved`. This is a terminal lifecycle state, not a + triage role from `triage-labels.md`. +- Under `## Comments`, append the implementation commit IDs and a concise + verification note. +- Commit the ticket update separately so the recorded commit IDs are exact. + ## When a skill says "publish to the issue tracker" Create a new file under `.scratch//` (creating the directory if needed). From 0bd719c3266eee3a870793713c87419f35fc3224 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 10:52:14 -0400 Subject: [PATCH 012/126] feat(evaluation): freeze immutable benchmark snapshot --- README.md | 19 +++ tests/test_evaluation_cli.py | 194 +++++++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 25 ++++ vision_aid/evaluation/snapshot.py | 196 ++++++++++++++++++++++++++++++ 4 files changed, 434 insertions(+) create mode 100644 vision_aid/evaluation/snapshot.py diff --git a/README.md b/README.md index bfb239d..57c5863 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,25 @@ place it in the repository root, and initialize the private workspace: uv run visionaid-evaluate prepare ``` +To freeze the Pristine homepage for model comparisons, supply its authorized +URL and record the stakeholder-supported temporal assumption during preparation: + +```bash +uv run visionaid-evaluate prepare \ + --snapshot-url "$PRISTINE_HOMEPAGE_URL" \ + --temporal-assumption \ + "Stakeholders report that the homepage has not changed since the workbook audit." +``` + +The command saves the HTTP response body without HTML transformation as +`.model-evaluation/snapshots/pristine-homepage.html`. Its adjacent JSON file +records the source and final URLs, retrieval time, HTTP status and identity +headers, byte length, SHA-256, and temporal assumption. Repeating preparation +for the same URL verifies and reuses those files without another request. +Checksum drift, changed provenance, or a partial snapshot fails closed; prepare +never silently replaces an existing benchmark. Local HTML files, including the +unrelated DAT Vision Aid fixture, cannot be used as the Pristine snapshot. + Use `--workbook PATH` for another authorized local copy outside the repository. The fixed `.model-evaluation/` workspace contains separate `references/`, `reviews/`, `snapshots/`, `runs/`, and `reports/` partitions, all covered by one diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index ac0d8bf..0dc8079 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,12 +1,66 @@ +import json import os import subprocess import sys +from contextlib import contextmanager +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from tempfile import TemporaryDirectory +from threading import Thread +from typing import Iterator import pytest PROJECT_ROOT = Path(__file__).parents[1] +SNAPSHOT_BODY = ( + b'\r\nPristine' + b"caf\xc3\xa9\r\n" +) +SNAPSHOT_SHA256 = "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" +TEMPORAL_ASSUMPTION = ( + "Stakeholders report that the homepage has not changed since the workbook audit." +) + + +class SnapshotHTTPServer(ThreadingHTTPServer): + """Serve one synthetic HTML response and record benchmark fetches.""" + + request_count = 0 + + +class SnapshotHandler(BaseHTTPRequestHandler): + """Return a byte-sensitive local response without external network access.""" + + def do_GET(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + server = self.server + assert isinstance(server, SnapshotHTTPServer) + server.request_count += 1 + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(SNAPSHOT_BODY))) + self.send_header("ETag", '"synthetic-v1"') + self.send_header("Last-Modified", "Wed, 16 Sep 2026 14:00:00 GMT") + self.end_headers() + self.wfile.write(SNAPSHOT_BODY) + + def log_message(self, format: str, *args: object) -> None: + """Keep the test output free of local server access logs.""" + + +@contextmanager +def local_snapshot_server() -> Iterator[tuple[str, SnapshotHTTPServer]]: + """Run the synthetic benchmark source on an ephemeral loopback port.""" + server = SnapshotHTTPServer(("127.0.0.1", 0), SnapshotHandler) + thread = Thread(target=server.serve_forever) + thread.start() + host, port = server.server_address + try: + yield f"http://{host}:{port}/pristine-homepage", server + finally: + server.shutdown() + thread.join() + server.server_close() def run_cli( @@ -125,6 +179,146 @@ def test_prepare_rejects_an_unignored_workbook_inside_the_worktree( assert "outside the repository" in result.stderr +def test_prepare_captures_exact_html_with_snapshot_provenance(tmp_path: Path) -> None: + """Prepare freezes the HTTP body consumed by the audit pipeline.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + + with local_snapshot_server() as (source_url, server): + result = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + source_url, + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + + assert result.returncode == 0, result.stderr + assert "Benchmark snapshot captured" in result.stdout + assert server.request_count == 1 + + snapshot_directory = tmp_path / ".model-evaluation" / "snapshots" + html_path = snapshot_directory / "pristine-homepage.html" + metadata_path = snapshot_directory / "pristine-homepage.json" + assert html_path.read_bytes() == SNAPSHOT_BODY + + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert metadata["source_url"] == source_url + assert metadata["sha256"] == SNAPSHOT_SHA256 + assert metadata["byte_length"] == len(SNAPSHOT_BODY) + assert metadata["temporal_assumption"] == TEMPORAL_ASSUMPTION + assert metadata["html_file"] == html_path.name + assert metadata["http"] == { + "content_length": str(len(SNAPSHOT_BODY)), + "content_type": "text/html; charset=utf-8", + "etag": '"synthetic-v1"', + "final_url": source_url, + "last_modified": "Wed, 16 Sep 2026 14:00:00 GMT", + "status_code": 200, + } + retrieved_at = datetime.fromisoformat( + metadata["retrieved_at"].replace("Z", "+00:00") + ) + assert retrieved_at.tzinfo is not None + + +def test_prepare_reuses_a_verified_snapshot_without_refetching(tmp_path: Path) -> None: + """An existing benchmark is verified in place and never fetched again.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + + with local_snapshot_server() as (source_url, server): + first = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + source_url, + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + second = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + source_url, + cwd=tmp_path, + ) + + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert "Benchmark snapshot reused" in second.stdout + assert server.request_count == 1 + + +def test_prepare_refuses_to_overwrite_a_changed_snapshot(tmp_path: Path) -> None: + """Checksum drift fails closed without a network request or overwrite.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + + with local_snapshot_server() as (source_url, server): + first = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + source_url, + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + assert first.returncode == 0, first.stderr + html_path = ( + tmp_path + / ".model-evaluation" + / "snapshots" + / "pristine-homepage.html" + ) + html_path.write_bytes(b"changed benchmark") + + second = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + source_url, + cwd=tmp_path, + ) + + assert second.returncode == 2 + assert "checksum does not match" in second.stderr + assert "will not be refetched or overwritten" in second.stderr + assert html_path.read_bytes() == b"changed benchmark" + assert server.request_count == 1 + + +def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( + tmp_path: Path, +) -> None: + """The unrelated committed fixture cannot stand in for Pristine.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + + result = run_cli( + "prepare", + "--workbook", + str(workbook), + "--snapshot-url", + str(PROJECT_ROOT / "test_files" / "dat_visionaid_home.html"), + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + + assert result.returncode == 2 + assert "DAT Vision Aid fixture is not a Pristine benchmark input" in result.stderr + + @pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( tmp_path: Path, api_key_name: str diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index b3a9f98..636795f 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Sequence from pathlib import Path +from vision_aid.evaluation.snapshot import prepare_benchmark_snapshot from vision_aid.evaluation.workspace import ( DEFAULT_WORKBOOK_FILENAME, PrivateInputError, @@ -41,6 +42,17 @@ def build_parser() -> argparse.ArgumentParser: f"authorized local source workbook (default: ./{DEFAULT_WORKBOOK_FILENAME})" ), ) + prepare.add_argument( + "--snapshot-url", + help="HTTP(S) source URL for an immutable Pristine homepage capture", + ) + prepare.add_argument( + "--temporal-assumption", + help=( + "stakeholder-supported statement relating a new capture to the " + "workbook audit" + ), + ) prepare.set_defaults(handler=_prepare) audit = workflows.add_parser( @@ -70,6 +82,19 @@ def _prepare(arguments: argparse.Namespace) -> int: workspace.initialize() print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") print(f"Private source workbook: {workbook}") + if arguments.temporal_assumption and not arguments.snapshot_url: + raise PrivateInputError( + "--temporal-assumption requires --snapshot-url." + ) + if arguments.snapshot_url: + snapshot = prepare_benchmark_snapshot( + workspace, + source_url=arguments.snapshot_url, + temporal_assumption=arguments.temporal_assumption, + ) + action = "reused" if snapshot.reused else "captured" + print(f"Benchmark snapshot {action}: {snapshot.html_path.resolve()}") + print(f"Benchmark SHA-256: {snapshot.metadata['sha256']}") return 0 diff --git a/vision_aid/evaluation/snapshot.py b/vision_aid/evaluation/snapshot.py new file mode 100644 index 0000000..57c2bd4 --- /dev/null +++ b/vision_aid/evaluation/snapshot.py @@ -0,0 +1,196 @@ +"""Immutable benchmark snapshots for model evaluation.""" + +import hashlib +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from vision_aid.evaluation.workspace import PrivateInputError, PrivateWorkspace + +SNAPSHOT_HTML_FILENAME = "pristine-homepage.html" +SNAPSHOT_METADATA_FILENAME = "pristine-homepage.json" +DAT_VISION_AID_FIXTURE = "dat_visionaid_home.html" +HTTP_TIMEOUT = (5, 30) + + +@dataclass(frozen=True) +class BenchmarkSnapshot: + """A verified raw HTML benchmark and its provenance record.""" + + html_path: Path + metadata_path: Path + metadata: dict[str, Any] + reused: bool + + +def prepare_benchmark_snapshot( + workspace: PrivateWorkspace, + source_url: str, + temporal_assumption: str | None, +) -> BenchmarkSnapshot: + """Capture a benchmark once or verify an existing immutable capture.""" + _require_http_source(source_url) + html_path = workspace.root / "snapshots" / SNAPSHOT_HTML_FILENAME + metadata_path = workspace.root / "snapshots" / SNAPSHOT_METADATA_FILENAME + + if html_path.exists() or metadata_path.exists(): + return _load_snapshot( + html_path, + metadata_path, + source_url, + temporal_assumption, + ) + + assumption = (temporal_assumption or "").strip() + if not assumption: + raise PrivateInputError( + "A new benchmark snapshot requires --temporal-assumption so the " + "stakeholder-supported relationship to the workbook audit is explicit." + ) + + import requests + + try: + response = requests.get(source_url, timeout=HTTP_TIMEOUT) + except requests.RequestException as error: + raise PrivateInputError( + f"Could not retrieve Pristine benchmark source {source_url}: {error}" + ) from error + + if response.status_code != 200: + raise PrivateInputError( + f"Pristine benchmark source returned HTTP {response.status_code}; " + "no snapshot was saved." + ) + content_type = response.headers.get("Content-Type") + if content_type is not None and not content_type.lower().startswith("text/html"): + raise PrivateInputError( + f"Pristine benchmark source returned {content_type!r}, not raw HTML; " + "no snapshot was saved." + ) + body = response.content + if not body: + raise PrivateInputError( + "Pristine benchmark source returned an empty body; no snapshot was saved." + ) + + metadata = { + "byte_length": len(body), + "html_file": SNAPSHOT_HTML_FILENAME, + "http": { + "content_length": response.headers.get("Content-Length"), + "content_type": content_type, + "etag": response.headers.get("ETag"), + "final_url": response.url, + "last_modified": response.headers.get("Last-Modified"), + "status_code": response.status_code, + }, + "retrieved_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "sha256": hashlib.sha256(body).hexdigest(), + "source_url": source_url, + "temporal_assumption": assumption, + } + _save_new_snapshot(html_path, metadata_path, body, metadata) + return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=False) + + +def _require_http_source(source_url: str) -> None: + """Reject local substitutes and malformed benchmark source URLs.""" + parsed = urlparse(source_url) + if parsed.scheme in {"http", "https"} and parsed.netloc: + return + if Path(source_url).name == DAT_VISION_AID_FIXTURE: + raise PrivateInputError( + "The DAT Vision Aid fixture is not a Pristine benchmark input. " + "Capture the authorized Pristine homepage over HTTP(S)." + ) + raise PrivateInputError( + "Benchmark snapshot sources must be HTTP(S) URLs. Local HTML files " + "cannot substitute for the Pristine homepage." + ) + + +def _load_snapshot( + html_path: Path, + metadata_path: Path, + source_url: str, + temporal_assumption: str | None, +) -> BenchmarkSnapshot: + """Validate an existing snapshot without fetching or changing it.""" + if not html_path.is_file() or not metadata_path.is_file(): + raise PrivateInputError( + "The benchmark snapshot is incomplete. It will not be refetched or " + "overwritten; restore the matching HTML and metadata files." + ) + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PrivateInputError( + "The benchmark snapshot metadata is unreadable. The existing " + "snapshot will not be overwritten." + ) from error + if not isinstance(metadata, dict): + raise PrivateInputError( + "The benchmark snapshot metadata is invalid. The existing snapshot " + "will not be overwritten." + ) + + recorded_url = metadata.get("source_url") + if recorded_url != source_url: + raise PrivateInputError( + f"The existing benchmark snapshot was captured from {recorded_url!r}, " + f"not {source_url!r}. It will not be refetched or overwritten." + ) + supplied_assumption = (temporal_assumption or "").strip() + if ( + supplied_assumption + and supplied_assumption != metadata.get("temporal_assumption") + ): + raise PrivateInputError( + "The supplied temporal assumption differs from the immutable snapshot " + "metadata. The existing snapshot will not be overwritten." + ) + + body = html_path.read_bytes() + actual_sha256 = hashlib.sha256(body).hexdigest() + if metadata.get("sha256") != actual_sha256: + raise PrivateInputError( + "The benchmark snapshot checksum does not match its metadata. The " + "existing snapshot will not be refetched or overwritten." + ) + if metadata.get("byte_length") != len(body): + raise PrivateInputError( + "The benchmark snapshot byte length does not match its metadata. The " + "existing snapshot will not be refetched or overwritten." + ) + if metadata.get("html_file") != html_path.name: + raise PrivateInputError( + "The benchmark metadata identifies a different HTML file. The " + "existing snapshot will not be refetched or overwritten." + ) + return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=True) + + +def _save_new_snapshot( + html_path: Path, + metadata_path: Path, + body: bytes, + metadata: dict[str, Any], +) -> None: + """Create both snapshot artifacts exclusively so neither is overwritten.""" + metadata_bytes = ( + json.dumps(metadata, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + ).encode("utf-8") + try: + with html_path.open("xb") as html_file: + html_file.write(body) + with metadata_path.open("xb") as metadata_file: + metadata_file.write(metadata_bytes) + except FileExistsError as error: + raise PrivateInputError( + "A benchmark snapshot artifact appeared during capture. It was not " + "overwritten; rerun prepare to verify the existing snapshot." + ) from error From fc630cb6e813d2f35ea39c5a1affe316db1a234d Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 10:59:32 -0400 Subject: [PATCH 013/126] fix(evaluation): harden snapshot publication --- README.md | 10 +- tests/test_evaluation_cli.py | 162 +++++++++++++++----- vision_aid/evaluation/cli.py | 2 +- vision_aid/evaluation/snapshot.py | 247 +++++++++++++++++++++++------- 4 files changed, 321 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 57c5863..1d8bce5 100644 --- a/README.md +++ b/README.md @@ -260,10 +260,12 @@ uv run visionaid-evaluate prepare \ ``` The command saves the HTTP response body without HTML transformation as -`.model-evaluation/snapshots/pristine-homepage.html`. Its adjacent JSON file -records the source and final URLs, retrieval time, HTTP status and identity -headers, byte length, SHA-256, and temporal assumption. Repeating preparation -for the same URL verifies and reuses those files without another request. +`.model-evaluation/snapshots/pristine-homepage/source.html`. The adjacent +`metadata.json` records the source and final URLs, retrieval time, HTTP status, +request user agent, response identity headers, byte length, SHA-256, and +temporal assumption. Both files are staged and published as one immutable +bundle. Repeating preparation for the same URL verifies and reuses that bundle +without another request. Checksum drift, changed provenance, or a partial snapshot fails closed; prepare never silently replaces an existing benchmark. Local HTML files, including the unrelated DAT Vision Aid fixture, cannot be used as the Pristine snapshot. diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 0dc8079..6207136 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory from threading import Thread -from typing import Iterator +from typing import Callable, Iterator import pytest @@ -27,31 +27,46 @@ class SnapshotHTTPServer(ThreadingHTTPServer): """Serve one synthetic HTML response and record benchmark fetches.""" request_count = 0 + request_user_agents: list[str | None] + on_request: Callable[[], None] | None class SnapshotHandler(BaseHTTPRequestHandler): """Return a byte-sensitive local response without external network access.""" def do_GET(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + """Serve the synthetic homepage and record acquisition metadata.""" server = self.server assert isinstance(server, SnapshotHTTPServer) server.request_count += 1 + server.request_user_agents.append(self.headers.get("User-Agent")) + if server.on_request is not None: + server.on_request() + body = SNAPSHOT_BODY + content_type = "text/html; charset=utf-8" + if self.path == "/non-utf8": + body = b"caf\xe9" + content_type = "text/html; charset=iso-8859-1" self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(SNAPSHOT_BODY))) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) self.send_header("ETag", '"synthetic-v1"') self.send_header("Last-Modified", "Wed, 16 Sep 2026 14:00:00 GMT") self.end_headers() - self.wfile.write(SNAPSHOT_BODY) + self.wfile.write(body) def log_message(self, format: str, *args: object) -> None: """Keep the test output free of local server access logs.""" @contextmanager -def local_snapshot_server() -> Iterator[tuple[str, SnapshotHTTPServer]]: +def local_snapshot_server( + on_request: Callable[[], None] | None = None, +) -> Iterator[tuple[str, SnapshotHTTPServer]]: """Run the synthetic benchmark source on an ephemeral loopback port.""" server = SnapshotHTTPServer(("127.0.0.1", 0), SnapshotHandler) + server.request_user_agents = [] + server.on_request = on_request thread = Thread(target=server.serve_forever) thread.start() host, port = server.server_address @@ -82,6 +97,14 @@ def run_cli( ) +@pytest.fixture +def synthetic_workbook(tmp_path: Path) -> Path: + """Create the minimal authorized workbook placeholder used by CLI tests.""" + workbook = tmp_path / "synthetic-reference.xlsx" + workbook.write_bytes(b"synthetic workbook placeholder") + return workbook + + def test_help_exposes_the_evaluation_workflows() -> None: """The CLI makes every evaluation lifecycle workflow discoverable.""" result = run_cli("--help") @@ -91,22 +114,23 @@ def test_help_exposes_the_evaluation_workflows() -> None: assert workflow in result.stdout -def test_prepare_initializes_one_partitioned_private_workspace(tmp_path: Path) -> None: +def test_prepare_initializes_one_partitioned_private_workspace( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: """Prepare creates all private artifact partitions around a local workbook.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") workspace = tmp_path / ".model-evaluation" result = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), cwd=tmp_path, ) assert result.returncode == 0, result.stderr assert "Private evaluation workspace initialized" in result.stdout - assert str(workbook.resolve()) in result.stdout + assert str(synthetic_workbook.resolve()) in result.stdout assert sorted(path.name for path in workspace.iterdir()) == [ "references", "reports", @@ -179,16 +203,16 @@ def test_prepare_rejects_an_unignored_workbook_inside_the_worktree( assert "outside the repository" in result.stderr -def test_prepare_captures_exact_html_with_snapshot_provenance(tmp_path: Path) -> None: +def test_prepare_captures_exact_html_with_snapshot_provenance( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: """Prepare freezes the HTTP body consumed by the audit pipeline.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - with local_snapshot_server() as (source_url, server): result = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", source_url, "--temporal-assumption", @@ -199,10 +223,13 @@ def test_prepare_captures_exact_html_with_snapshot_provenance(tmp_path: Path) -> assert result.returncode == 0, result.stderr assert "Benchmark snapshot captured" in result.stdout assert server.request_count == 1 + assert server.request_user_agents == ["Mozilla/5.0"] - snapshot_directory = tmp_path / ".model-evaluation" / "snapshots" - html_path = snapshot_directory / "pristine-homepage.html" - metadata_path = snapshot_directory / "pristine-homepage.json" + snapshot_directory = ( + tmp_path / ".model-evaluation" / "snapshots" / "pristine-homepage" + ) + html_path = snapshot_directory / "source.html" + metadata_path = snapshot_directory / "metadata.json" assert html_path.read_bytes() == SNAPSHOT_BODY metadata = json.loads(metadata_path.read_text(encoding="utf-8")) @@ -217,6 +244,7 @@ def test_prepare_captures_exact_html_with_snapshot_provenance(tmp_path: Path) -> "etag": '"synthetic-v1"', "final_url": source_url, "last_modified": "Wed, 16 Sep 2026 14:00:00 GMT", + "request_user_agent": "Mozilla/5.0", "status_code": 200, } retrieved_at = datetime.fromisoformat( @@ -225,16 +253,16 @@ def test_prepare_captures_exact_html_with_snapshot_provenance(tmp_path: Path) -> assert retrieved_at.tzinfo is not None -def test_prepare_reuses_a_verified_snapshot_without_refetching(tmp_path: Path) -> None: +def test_prepare_reuses_a_verified_snapshot_without_refetching( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: """An existing benchmark is verified in place and never fetched again.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - with local_snapshot_server() as (source_url, server): first = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", source_url, "--temporal-assumption", @@ -244,7 +272,7 @@ def test_prepare_reuses_a_verified_snapshot_without_refetching(tmp_path: Path) - second = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", source_url, cwd=tmp_path, @@ -256,16 +284,39 @@ def test_prepare_reuses_a_verified_snapshot_without_refetching(tmp_path: Path) - assert server.request_count == 1 -def test_prepare_refuses_to_overwrite_a_changed_snapshot(tmp_path: Path) -> None: - """Checksum drift fails closed without a network request or overwrite.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") +def test_prepare_rejects_html_the_audit_pipeline_cannot_decode( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: + """A snapshot must satisfy the pipeline's UTF-8 file contract.""" + with local_snapshot_server() as (source_url, _server): + result = run_cli( + "prepare", + "--workbook", + str(synthetic_workbook), + "--snapshot-url", + source_url.replace("/pristine-homepage", "/non-utf8"), + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + + assert result.returncode == 2 + assert "not valid UTF-8" in result.stderr + snapshot_directory = tmp_path / ".model-evaluation" / "snapshots" + assert list(snapshot_directory.iterdir()) == [] + +def test_prepare_refuses_to_overwrite_a_changed_snapshot( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: + """Checksum drift fails closed without a network request or overwrite.""" with local_snapshot_server() as (source_url, server): first = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", source_url, "--temporal-assumption", @@ -277,14 +328,15 @@ def test_prepare_refuses_to_overwrite_a_changed_snapshot(tmp_path: Path) -> None tmp_path / ".model-evaluation" / "snapshots" - / "pristine-homepage.html" + / "pristine-homepage" + / "source.html" ) html_path.write_bytes(b"changed benchmark") second = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", source_url, cwd=tmp_path, @@ -297,17 +349,49 @@ def test_prepare_refuses_to_overwrite_a_changed_snapshot(tmp_path: Path) -> None assert server.request_count == 1 +def test_prepare_publishes_the_snapshot_as_one_immutable_bundle( + tmp_path: Path, + synthetic_workbook: Path, +) -> None: + """A competing snapshot cannot be mixed with a capture in progress.""" + snapshot_directory = ( + tmp_path / ".model-evaluation" / "snapshots" / "pristine-homepage" + ) + + def publish_competing_snapshot() -> None: + """Simulate another prepare process winning the publication race.""" + snapshot_directory.mkdir() + (snapshot_directory / "race-marker").write_text( + "winner", + encoding="utf-8", + ) + + with local_snapshot_server(publish_competing_snapshot) as (source_url, _server): + result = run_cli( + "prepare", + "--workbook", + str(synthetic_workbook), + "--snapshot-url", + source_url, + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + + assert result.returncode == 2 + assert "appeared during capture" in result.stderr + assert [path.name for path in snapshot_directory.iterdir()] == ["race-marker"] + + def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( tmp_path: Path, + synthetic_workbook: Path, ) -> None: """The unrelated committed fixture cannot stand in for Pristine.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - result = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), "--snapshot-url", str(PROJECT_ROOT / "test_files" / "dat_visionaid_home.html"), "--temporal-assumption", @@ -321,15 +405,15 @@ def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( @pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( - tmp_path: Path, api_key_name: str + tmp_path: Path, + synthetic_workbook: Path, + api_key_name: str, ) -> None: """Environment credentials alone can never make evaluation billable.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") prepared = run_cli( "prepare", "--workbook", - str(workbook), + str(synthetic_workbook), cwd=tmp_path, ) assert prepared.returncode == 0, prepared.stderr diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 636795f..6c6dd7e 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -94,7 +94,7 @@ def _prepare(arguments: argparse.Namespace) -> int: ) action = "reused" if snapshot.reused else "captured" print(f"Benchmark snapshot {action}: {snapshot.html_path.resolve()}") - print(f"Benchmark SHA-256: {snapshot.metadata['sha256']}") + print(f"Benchmark SHA-256: {snapshot.metadata.sha256}") return 0 diff --git a/vision_aid/evaluation/snapshot.py b/vision_aid/evaluation/snapshot.py index 57c2bd4..a896de5 100644 --- a/vision_aid/evaluation/snapshot.py +++ b/vision_aid/evaluation/snapshot.py @@ -2,18 +2,115 @@ import hashlib import json -from dataclasses import dataclass +from dataclasses import asdict, dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Any +from tempfile import mkdtemp +from typing import cast from urllib.parse import urlparse from vision_aid.evaluation.workspace import PrivateInputError, PrivateWorkspace -SNAPSHOT_HTML_FILENAME = "pristine-homepage.html" -SNAPSHOT_METADATA_FILENAME = "pristine-homepage.json" +SNAPSHOT_DIRECTORY_NAME = "pristine-homepage" +SNAPSHOT_HTML_FILENAME = "source.html" +SNAPSHOT_METADATA_FILENAME = "metadata.json" DAT_VISION_AID_FIXTURE = "dat_visionaid_home.html" HTTP_TIMEOUT = (5, 30) +PIPELINE_USER_AGENT = "Mozilla/5.0" + + +@dataclass(frozen=True) +class SnapshotHTTPMetadata: + """HTTP request and response details that identify captured content.""" + + content_length: str | None + content_type: str | None + etag: str | None + final_url: str + last_modified: str | None + request_user_agent: str + status_code: int + + +@dataclass(frozen=True) +class SnapshotMetadata: + """Required provenance for one immutable benchmark snapshot.""" + + byte_length: int + html_file: str + http: SnapshotHTTPMetadata + retrieved_at: str + sha256: str + source_url: str + temporal_assumption: str + + def to_json(self) -> object: + """Return the JSON-compatible representation stored beside the HTML.""" + return asdict(self) + + @classmethod + def from_json(cls, value: object) -> "SnapshotMetadata": + """Validate and construct snapshot provenance loaded from JSON.""" + if not isinstance(value, dict): + raise ValueError("snapshot metadata must be a JSON object") + http_value = value.get("http") + if not isinstance(http_value, dict): + raise ValueError("snapshot HTTP metadata must be a JSON object") + + required_strings = ( + "html_file", + "retrieved_at", + "sha256", + "source_url", + "temporal_assumption", + ) + if any(not isinstance(value.get(name), str) for name in required_strings): + raise ValueError("snapshot metadata has a missing string field") + required_http_strings = ( + "final_url", + "request_user_agent", + ) + if any( + not isinstance(http_value.get(name), str) + for name in required_http_strings + ): + raise ValueError("snapshot HTTP metadata has a missing string field") + optional_http_strings = ( + "content_length", + "content_type", + "etag", + "last_modified", + ) + if any( + http_value.get(name) is not None + and not isinstance(http_value.get(name), str) + for name in optional_http_strings + ): + raise ValueError("snapshot HTTP metadata has an invalid header field") + byte_length = value.get("byte_length") + status_code = http_value.get("status_code") + if not isinstance(byte_length, int) or isinstance(byte_length, bool): + raise ValueError("snapshot byte length must be an integer") + if not isinstance(status_code, int) or isinstance(status_code, bool): + raise ValueError("snapshot HTTP status must be an integer") + + return cls( + byte_length=byte_length, + html_file=cast(str, value["html_file"]), + http=SnapshotHTTPMetadata( + content_length=cast(str | None, http_value.get("content_length")), + content_type=cast(str | None, http_value.get("content_type")), + etag=cast(str | None, http_value.get("etag")), + final_url=cast(str, http_value["final_url"]), + last_modified=cast(str | None, http_value.get("last_modified")), + request_user_agent=cast(str, http_value["request_user_agent"]), + status_code=status_code, + ), + retrieved_at=cast(str, value["retrieved_at"]), + sha256=cast(str, value["sha256"]), + source_url=cast(str, value["source_url"]), + temporal_assumption=cast(str, value["temporal_assumption"]), + ) @dataclass(frozen=True) @@ -22,7 +119,7 @@ class BenchmarkSnapshot: html_path: Path metadata_path: Path - metadata: dict[str, Any] + metadata: SnapshotMetadata reused: bool @@ -33,10 +130,13 @@ def prepare_benchmark_snapshot( ) -> BenchmarkSnapshot: """Capture a benchmark once or verify an existing immutable capture.""" _require_http_source(source_url) - html_path = workspace.root / "snapshots" / SNAPSHOT_HTML_FILENAME - metadata_path = workspace.root / "snapshots" / SNAPSHOT_METADATA_FILENAME + snapshot_directory = ( + workspace.root / "snapshots" / SNAPSHOT_DIRECTORY_NAME + ) + html_path = snapshot_directory / SNAPSHOT_HTML_FILENAME + metadata_path = snapshot_directory / SNAPSHOT_METADATA_FILENAME - if html_path.exists() or metadata_path.exists(): + if snapshot_directory.exists(): return _load_snapshot( html_path, metadata_path, @@ -54,7 +154,11 @@ def prepare_benchmark_snapshot( import requests try: - response = requests.get(source_url, timeout=HTTP_TIMEOUT) + response = requests.get( + source_url, + headers={"User-Agent": PIPELINE_USER_AGENT}, + timeout=HTTP_TIMEOUT, + ) except requests.RequestException as error: raise PrivateInputError( f"Could not retrieve Pristine benchmark source {source_url}: {error}" @@ -76,24 +180,32 @@ def prepare_benchmark_snapshot( raise PrivateInputError( "Pristine benchmark source returned an empty body; no snapshot was saved." ) + try: + body.decode("utf-8") + except UnicodeDecodeError as error: + raise PrivateInputError( + "Pristine benchmark HTML is not valid UTF-8 and cannot be consumed " + "by the existing audit pipeline; no snapshot was saved." + ) from error - metadata = { - "byte_length": len(body), - "html_file": SNAPSHOT_HTML_FILENAME, - "http": { - "content_length": response.headers.get("Content-Length"), - "content_type": content_type, - "etag": response.headers.get("ETag"), - "final_url": response.url, - "last_modified": response.headers.get("Last-Modified"), - "status_code": response.status_code, - }, - "retrieved_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), - "sha256": hashlib.sha256(body).hexdigest(), - "source_url": source_url, - "temporal_assumption": assumption, - } - _save_new_snapshot(html_path, metadata_path, body, metadata) + metadata = SnapshotMetadata( + byte_length=len(body), + html_file=SNAPSHOT_HTML_FILENAME, + http=SnapshotHTTPMetadata( + content_length=response.headers.get("Content-Length"), + content_type=content_type, + etag=response.headers.get("ETag"), + final_url=response.url, + last_modified=response.headers.get("Last-Modified"), + request_user_agent=PIPELINE_USER_AGENT, + status_code=response.status_code, + ), + retrieved_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"), + sha256=hashlib.sha256(body).hexdigest(), + source_url=source_url, + temporal_assumption=assumption, + ) + _save_new_snapshot(snapshot_directory, body, metadata) return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=False) @@ -126,47 +238,49 @@ def _load_snapshot( "overwritten; restore the matching HTML and metadata files." ) try: - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError) as error: + metadata = SnapshotMetadata.from_json( + json.loads(metadata_path.read_text(encoding="utf-8")) + ) + except (OSError, UnicodeError, ValueError) as error: raise PrivateInputError( "The benchmark snapshot metadata is unreadable. The existing " "snapshot will not be overwritten." ) from error - if not isinstance(metadata, dict): - raise PrivateInputError( - "The benchmark snapshot metadata is invalid. The existing snapshot " - "will not be overwritten." - ) - - recorded_url = metadata.get("source_url") - if recorded_url != source_url: + if metadata.source_url != source_url: raise PrivateInputError( - f"The existing benchmark snapshot was captured from {recorded_url!r}, " + "The existing benchmark snapshot was captured from " + f"{metadata.source_url!r}, " f"not {source_url!r}. It will not be refetched or overwritten." ) supplied_assumption = (temporal_assumption or "").strip() if ( supplied_assumption - and supplied_assumption != metadata.get("temporal_assumption") + and supplied_assumption != metadata.temporal_assumption ): raise PrivateInputError( "The supplied temporal assumption differs from the immutable snapshot " "metadata. The existing snapshot will not be overwritten." ) - body = html_path.read_bytes() + try: + body = html_path.read_bytes() + except OSError as error: + raise PrivateInputError( + "The benchmark snapshot HTML is unreadable. The existing snapshot " + "will not be overwritten." + ) from error actual_sha256 = hashlib.sha256(body).hexdigest() - if metadata.get("sha256") != actual_sha256: + if metadata.sha256 != actual_sha256: raise PrivateInputError( "The benchmark snapshot checksum does not match its metadata. The " "existing snapshot will not be refetched or overwritten." ) - if metadata.get("byte_length") != len(body): + if metadata.byte_length != len(body): raise PrivateInputError( "The benchmark snapshot byte length does not match its metadata. The " "existing snapshot will not be refetched or overwritten." ) - if metadata.get("html_file") != html_path.name: + if metadata.html_file != html_path.name: raise PrivateInputError( "The benchmark metadata identifies a different HTML file. The " "existing snapshot will not be refetched or overwritten." @@ -175,22 +289,43 @@ def _load_snapshot( def _save_new_snapshot( - html_path: Path, - metadata_path: Path, + snapshot_directory: Path, body: bytes, - metadata: dict[str, Any], + metadata: SnapshotMetadata, ) -> None: - """Create both snapshot artifacts exclusively so neither is overwritten.""" + """Stage both artifacts and publish their directory as one atomic unit.""" metadata_bytes = ( - json.dumps(metadata, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + json.dumps( + metadata.to_json(), + indent=2, + sort_keys=True, + ensure_ascii=False, + ) + + "\n" ).encode("utf-8") + staged_directory = Path( + mkdtemp( + prefix=f".{SNAPSHOT_DIRECTORY_NAME}-", + dir=snapshot_directory.parent, + ) + ) try: - with html_path.open("xb") as html_file: - html_file.write(body) - with metadata_path.open("xb") as metadata_file: - metadata_file.write(metadata_bytes) - except FileExistsError as error: - raise PrivateInputError( - "A benchmark snapshot artifact appeared during capture. It was not " - "overwritten; rerun prepare to verify the existing snapshot." - ) from error + (staged_directory / SNAPSHOT_HTML_FILENAME).write_bytes(body) + (staged_directory / SNAPSHOT_METADATA_FILENAME).write_bytes(metadata_bytes) + if snapshot_directory.exists(): + raise FileExistsError(snapshot_directory) + staged_directory.rename(snapshot_directory) + except OSError as error: + if snapshot_directory.exists(): + message = ( + "A benchmark snapshot appeared during capture. It was not " + "overwritten; rerun prepare to verify the existing snapshot." + ) + else: + message = f"Could not save the benchmark snapshot: {error}" + raise PrivateInputError(message) from error + finally: + if staged_directory.exists(): + for staged_file in staged_directory.iterdir(): + staged_file.unlink() + staged_directory.rmdir() From e4f9a55eee7043d617736d236f0a512db6e9b496 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 10:59:49 -0400 Subject: [PATCH 014/126] docs(evaluation): resolve benchmark snapshot ticket --- .../03-freeze-pristine-benchmark-snapshot.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md index ed8a497..46a5caf 100644 --- a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md +++ b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md @@ -4,10 +4,17 @@ **Blocked by:** 02: Bootstrap a private, safe evaluation workspace. -**Status:** ready-for-agent +**Status:** resolved -- [ ] The preparation workflow saves the exact raw HTML consumed by the existing audit pipeline. -- [ ] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, SHA-256, and the stakeholder-supported temporal assumption. -- [ ] Reusing a snapshot does not refetch or silently overwrite it. -- [ ] The existing unrelated DAT Vision Aid fixture is rejected as a Pristine benchmark input. -- [ ] Tests use local HTTP fakes and verify content identity, provenance, hashing, and overwrite protection. +- [x] The preparation workflow saves the exact raw HTML consumed by the existing audit pipeline. +- [x] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, SHA-256, and the stakeholder-supported temporal assumption. +- [x] Reusing a snapshot does not refetch or silently overwrite it. +- [x] The existing unrelated DAT Vision Aid fixture is rejected as a Pristine benchmark input. +- [x] Tests use local HTTP fakes and verify content identity, provenance, hashing, and overwrite protection. + +## Comments + +- Implemented in `0bd719c` with review hardening in `fc630cb`. Verified by + local-HTTP snapshot tests covering byte identity, provenance, SHA-256, + no-refetch reuse, UTF-8 compatibility, atomic publication, fixture rejection, + and overwrite protection; the full suite passes with 17 tests. From 472500cd8c417f4ce3e422d4885ef9b2145203fa Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 12:09:58 -0400 Subject: [PATCH 015/126] feat(evaluation): complete guarded model evaluation workflow --- .github/workflows/ci.yml | 3 + README.md | 69 +++- processing_scripts/llm_client/client.py | 15 +- pyproject.toml | 9 +- tests/test_evaluation_audit.py | 241 ++++++++++++++ tests/test_match_review.py | 90 ++++++ tests/test_normalization.py | 102 ++++++ tests/test_programmatic_evaluation.py | 26 ++ tests/test_reference_set.py | 133 ++++++++ tests/test_scoring_and_reporting.py | 279 ++++++++++++++++ vision_aid/evaluation/audit.py | 411 ++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 193 ++++++++++- vision_aid/evaluation/matching.py | 369 +++++++++++++++++++++ vision_aid/evaluation/normalization.py | 201 ++++++++++++ vision_aid/evaluation/pricing.py | 71 ++++ vision_aid/evaluation/pricing.v1.json | 16 + vision_aid/evaluation/programmatic.py | 40 +++ vision_aid/evaluation/references.py | 366 +++++++++++++++++++++ vision_aid/evaluation/reporting.py | 90 ++++++ vision_aid/evaluation/schemas.py | 122 +++++++ vision_aid/evaluation/scoring.py | 244 ++++++++++++++ vision_aid/evaluation/serialization.py | 87 +++++ 22 files changed, 3162 insertions(+), 15 deletions(-) create mode 100644 tests/test_evaluation_audit.py create mode 100644 tests/test_match_review.py create mode 100644 tests/test_normalization.py create mode 100644 tests/test_programmatic_evaluation.py create mode 100644 tests/test_reference_set.py create mode 100644 tests/test_scoring_and_reporting.py create mode 100644 vision_aid/evaluation/audit.py create mode 100644 vision_aid/evaluation/matching.py create mode 100644 vision_aid/evaluation/normalization.py create mode 100644 vision_aid/evaluation/pricing.py create mode 100644 vision_aid/evaluation/pricing.v1.json create mode 100644 vision_aid/evaluation/programmatic.py create mode 100644 vision_aid/evaluation/references.py create mode 100644 vision_aid/evaluation/reporting.py create mode 100644 vision_aid/evaluation/schemas.py create mode 100644 vision_aid/evaluation/scoring.py create mode 100644 vision_aid/evaluation/serialization.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 774f495..f753ada 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,9 @@ jobs: print('all entry points import cleanly') " + - name: Network-free test suite + run: uv run pytest -q + # Exercises extractors, slicers, prompt templates and the programmatic # checkers end to end. --dry-run makes no API calls, so this needs no key # and costs nothing. diff --git a/README.md b/README.md index 1d8bce5..7010492 100644 --- a/README.md +++ b/README.md @@ -276,9 +276,71 @@ The fixed `.model-evaluation/` workspace contains separate `references/`, ignore rule. Run `uv run visionaid-evaluate --help` to discover the `prepare`, `audit`, -`review`, and `report` workflows. At this bootstrap milestone, `audit` is -always a network-free dry run—even when an API key is present—and live -execution is not exposed. +`review`, and `report` workflows. To import the homepage rows, record the +authorized workbook checksum and canonical homepage URL. This creates a +lossless private reference set and an Eligibility workbook: + +```bash +uv run visionaid-evaluate prepare \ + --workbook-sha256 "$AUTHORIZED_WORKBOOK_SHA256" \ + --homepage-url "$PRISTINE_HOMEPAGE_URL" +``` + +Complete every row in +`.model-evaluation/reviews/eligibility.xlsx`, including classification, +decision, rationale, reviewer, confidence, and timestamp, then validate it: + +```bash +uv run visionaid-evaluate review \ + --eligibility-workbook .model-evaluation/reviews/eligibility.xlsx +``` + +Only the four classifications `llm_eligible`, `programmatic`, +`unavailable_evidence`, and `ambiguous` are accepted. Global rows remain one +workbook-row scoring unit supported by homepage evidence. + +`audit` is network-free by default even if `OPENAI_API_KEY` exists. With an +approved reference set and benchmark snapshot it runs the production +extractors, filters, programmatic checks, slicers, and unchanged prompts to +write an exact Luna dry-run plan: + +```bash +uv run visionaid-evaluate audit +``` + +Review the request count, prompt hashes, estimated input usage, model +configuration, pricing identity, and authorization digest in the printed +summary. A billable run requires all of the following: `--live`, the exact +saved plan and digest, `OPENAI_API_KEY`, and a positive cost limit. One +authorization covers the summarized run and its policy-compliant retries: + +```bash +uv run visionaid-evaluate audit \ + --live \ + --plan .model-evaluation/runs/dry-run/evaluation-manifest.json \ + --authorize "$REVIEWED_AUTHORIZATION_DIGEST" \ + --max-audit-cost-usd "$APPROVED_LIMIT" +``` + +The runner uses `gpt-5.6-luna` through Chat Completions with medium reasoning, +omits temperature, keeps summaries disabled, and preserves every attempt and +raw response. It retries only transient timeout, rate-limit, and server +failures (at most twice). Exhausted failures make a run incomplete and +unrankable; successful malformed JSON is retained as a format failure and is +not retried. + +Canonical audit findings and programmatic findings are reviewed through a +private Matches workbook. Accepted matches require compatible page scope, +underlying failure, and location or element-group evidence; WCAG labels and +remediation wording are supporting evidence. The scorer is network-free, +enforces one-to-one matching and all required sub-defects, and emits JSON, CSV, +and Markdown from one score object. + +Workbook-row recall over approved LLM-eligible rows is the primary ranking +metric. Unrounded estimated audit cost is the only tie-breaker. Programmatic +and combined coverage, tokens, latency, evaluation cost, unmatched findings, +and format failures remain informational. The homepage result is illustrative +and cannot establish broad small-versus-large model equivalence. ## Running the Web App @@ -318,6 +380,7 @@ everything it does is free: - `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock - entry points import +- the complete synthetic pytest suite, with no private data or provider calls - a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed - `index.html`'s inline JavaScript parses - the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit diff --git a/processing_scripts/llm_client/client.py b/processing_scripts/llm_client/client.py index 9f0aae1..3092b55 100644 --- a/processing_scripts/llm_client/client.py +++ b/processing_scripts/llm_client/client.py @@ -8,10 +8,7 @@ import os from pathlib import Path -import anthropic -import openai from dotenv import load_dotenv -from google import genai # Project root is two levels up from this file (processing_scripts/llm_client/) _ENV_FILE = Path(__file__).resolve().parents[2] / ".env" @@ -81,9 +78,11 @@ def __init__( ): api_key = os.environ.get("ANTHROPIC_API_KEY") if not api_key: - raise EnvironmentError( + raise OSError( f"ANTHROPIC_API_KEY not found. Expected in {_ENV_FILE}" ) + import anthropic + self._client = anthropic.Anthropic(api_key=api_key) self.model = model self.temperature = temperature @@ -154,9 +153,11 @@ def __init__( ): api_key = os.environ.get("OPENAI_API_KEY") if not api_key: - raise EnvironmentError( + raise OSError( f"OPENAI_API_KEY not found. Expected in {_ENV_FILE}" ) + import openai + self._client = openai.OpenAI(api_key=api_key) self.model = model self.temperature = temperature @@ -223,9 +224,11 @@ def __init__( ): api_key = os.environ.get("GEMINI_API_KEY") if not api_key: - raise EnvironmentError( + raise OSError( f"GEMINI_API_KEY not found. Expected in {_ENV_FILE}" ) + from google import genai + self._client = genai.Client(api_key=api_key) self.model = model self.temperature = temperature diff --git a/pyproject.toml b/pyproject.toml index f99dfb9..c303171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,5 +30,10 @@ dev = [ [tool.pytest.ini_options] pythonpath = ["."] -[tool.setuptools] -packages = ["vision_aid", "vision_aid.evaluation"] +[tool.setuptools.packages.find] +include = ["vision_aid*", "processing_scripts*"] +namespaces = true + +[tool.setuptools.package-data] +"vision_aid.evaluation" = ["pricing.v1.json"] +"processing_scripts.llm" = ["*.txt"] diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py new file mode 100644 index 0000000..a5dc4b7 --- /dev/null +++ b/tests/test_evaluation_audit.py @@ -0,0 +1,241 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from vision_aid.evaluation.audit import ( + AuditExecutionError, + build_audit_plan, + execute_authorized_audit, +) +from vision_aid.evaluation.pricing import PriceSchedule + + +def _fake_pipeline( + html_path: str, + output_dir: Path, + api_key: str | None, + model: str, + dry_run: bool, + include_summaries: bool, + progress_callback=None, + request_config=None, +) -> dict: + assert api_key is None + assert dry_run is True + assert include_summaries is False + assert model == "gpt-5.6-luna" + assert request_config.reasoning_effort == "medium" + assert request_config.temperature is None + assert request_config.max_output_tokens == 8192 + prompts = output_dir / "prompts" + prompts.mkdir(parents=True) + entries = [] + for index, name in enumerate(("heading_structure", "link_clarity"), start=1): + prompt_text = f"prompt {index}" + (prompts / f"{name}.json").write_text( + json.dumps( + { + "prompt_name": name, + "checklist": "CL01", + "wcag_criteria": ["1.3.1"], + "prompt_index": index, + "prompt_tokens_est": 10 * index, + "prompt_text": prompt_text, + "payload_slice": "{}", + } + ), + encoding="utf-8", + ) + entries.append( + { + "name": name, + "checklist": "CL01", + "wcag_criteria": ["1.3.1"], + "prompt_tokens_est": 10 * index, + "status": "dry_run", + } + ) + (output_dir / "programmatic_findings.json").write_text("[]", encoding="utf-8") + return { + "html_file": html_path, + "model": model, + "dry_run": True, + "include_summaries": False, + "prompts_dry_run": entries, + "prompts_executed": [], + "prompts_skipped": [], + "pass1_filters_active": [], + } + + +def test_dry_plan_freezes_configuration_content_and_code_identities( + tmp_path: Path, +) -> None: + snapshot = tmp_path / "source.html" + snapshot.write_text("Synthetic", encoding="utf-8") + snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() + run_directory = tmp_path / "run" + + plan = build_audit_plan( + snapshot, + run_directory, + workbook_sha256="a" * 64, + snapshot_sha256=snapshot_sha256, + reference_set_version="references-v1", + eligibility_identity="c" * 64, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + + assert plan["configuration"] == { + "endpoint": "chat.completions", + "include_summaries": False, + "max_output_tokens": 8192, + "model": "gpt-5.6-luna", + "reasoning_effort": "medium", + "temperature": None, + "execution": "sequential", + } + assert plan["request_count"] == 2 + assert plan["estimated_input_tokens"] == 30 + assert list(plan["prompt_hashes"]) == ["heading_structure", "link_clarity"] + assert len(plan["parser_hash"]) == 64 + assert len(plan["pricing_identity"]) == 64 + assert len(plan["authorization_digest"]) == 64 + assert (run_directory / "evaluation-manifest.json").is_file() + + +class FakeClient: + def __init__(self, outcomes: list[dict]): + self.outcomes = outcomes + + def call(self, prompt: str) -> dict: + return self.outcomes.pop(0) + + +def _success(response: str, input_tokens: int = 10, output_tokens: int = 5) -> dict: + return { + "success": True, + "response": response, + "model": "gpt-5.6-luna", + "provider": {"name": "openai", "endpoint": "chat.completions"}, + "usage": { + "input_tokens": input_tokens, + "cached_input_tokens": 0, + "output_tokens": output_tokens, + "reasoning_tokens": 2, + "cache_creation_input_tokens": 0, + "total_tokens": input_tokens + output_tokens, + "provider_usage": {}, + }, + "stop_reason": "stop", + "duration_seconds": 0.2, + "failure": None, + } + + +def _transient() -> dict: + return { + "success": False, + "response": None, + "model": "gpt-5.6-luna", + "provider": {"name": "openai", "endpoint": "chat.completions"}, + "usage": { + "input_tokens": 2, + "cached_input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_creation_input_tokens": 0, + "total_tokens": 2, + "provider_usage": {}, + }, + "stop_reason": None, + "duration_seconds": 0.1, + "failure": {"type": "RateLimitError", "status_code": 429}, + "error": "rate limited", + } + + +def test_live_execution_requires_every_gate_and_records_retries_and_format_failures( + tmp_path: Path, +) -> None: + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() + run_directory = tmp_path / "run" + plan = build_audit_plan( + snapshot, + run_directory, + workbook_sha256="a" * 64, + snapshot_sha256=snapshot_sha256, + reference_set_version="references-v1", + eligibility_identity="c" * 64, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + plan_path = run_directory / "evaluation-manifest.json" + + with pytest.raises(AuditExecutionError, match="--live"): + execute_authorized_audit( + plan_path, + live=False, + authorization=plan["authorization_digest"], + api_key="secret", + max_audit_cost_usd="1", + client=FakeClient([]), + sleep=lambda _delay: None, + ) + with pytest.raises(AuditExecutionError, match="authorization"): + execute_authorized_audit( + plan_path, + live=True, + authorization="wrong", + api_key="secret", + max_audit_cost_usd="1", + client=FakeClient([]), + sleep=lambda _delay: None, + ) + + client = FakeClient( + [ + _transient(), + _success('[{"problem":"Heading skips a level"}]'), + _success("not json"), + ] + ) + manifest = execute_authorized_audit( + plan_path, + live=True, + authorization=plan["authorization_digest"], + api_key="secret", + max_audit_cost_usd="1", + client=client, + sleep=lambda _delay: None, + ) + + assert manifest["complete"] is True + assert manifest["rankable"] is True + assert len(manifest["requests"][0]["attempts"]) == 2 + assert manifest["format_failures"] == ["link_clarity"] + assert manifest["usage"]["input_tokens"] == 22 + assert manifest["estimated_audit_cost_usd"] == "0.0000164" + assert manifest["configuration"]["temperature"] is None + + +def test_versioned_price_schedule_uses_reported_token_categories() -> None: + schedule = PriceSchedule.default() + assert ( + schedule.estimate( + "gpt-5.6-luna", + { + "input_tokens": 100, + "cached_input_tokens": 40, + "cache_creation_input_tokens": 20, + "output_tokens": 10, + "reasoning_tokens": 5, + }, + ) + == "0.0000298" + ) diff --git a/tests/test_match_review.py b/tests/test_match_review.py new file mode 100644 index 0000000..32ff025 --- /dev/null +++ b/tests/test_match_review.py @@ -0,0 +1,90 @@ +from pathlib import Path + +import openpyxl + +from vision_aid.evaluation.matching import ( + HumanMatchReviewer, + export_match_workbook, + generate_candidates, +) +from vision_aid.evaluation.schemas import CanonicalFinding, ReferenceDefect + + +def _reference(identifier: str, scope: str = "home") -> ReferenceDefect: + return ReferenceDefect( + reference_id=identifier, + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + source_sheet="Defects", + source_row=2, + raw_evidence={"Problem": "Image problem"}, + canonical_url="https://example.test/", + page_scope=scope, # type: ignore[arg-type] + problem="Image has the wrong alternative text", + location="hero image", + wcag_evidence=("1.1.1",), + ) + + +def _finding(identifier: str, page: str = "https://example.test/") -> CanonicalFinding: + return CanonicalFinding( + finding_id=identifier, + source="audit", + run_id="run-1", + model="gpt-5.6-luna", + prompt="informative_alt_quality", + checklist="CL03", + page_url=page, + problem_family="informative_alt_quality", + problem="Alternative text does not describe the hero", + element="img", + location="hero image", + wcag_evidence=("1.1.1",), + raw_source={"problem": "Alternative text does not describe the hero"}, + parse_status="parsed", + ) + + +def test_candidate_generation_only_removes_hard_incompatibilities() -> None: + home = _reference("home") + global_reference = _reference("global", "global") + same_page = _finding("same") + other_page = _finding("other", "https://example.test/contact") + + candidates = generate_candidates((home, global_reference), (same_page, other_page)) + + assert ("home", "same") in candidates + assert ("home", "other") not in candidates + assert ("global", "same") in candidates + assert ("global", "other") in candidates + + +def test_human_match_review_round_trips_side_by_side_evidence(tmp_path: Path) -> None: + references = (_reference("ref1"),) + findings = (_finding("finding1"),) + review_path = tmp_path / "matches.xlsx" + export_match_workbook(references, findings, review_path) + + workbook = openpyxl.load_workbook(review_path) + sheet = workbook["Matches"] + columns = {cell.value: cell.column for cell in sheet[1]} + values = { + "decision": "accepted", + "page_compatible": True, + "failure_compatible": True, + "location_compatible": True, + "rationale": "Same alt-text failure on the hero image.", + "reviewer": "reviewer@example.test", + "confidence": 0.9, + "timestamp": "2026-09-19T12:00:00Z", + } + for name, value in values.items(): + sheet.cell(2, columns[name], value) + workbook.save(review_path) + + decisions = HumanMatchReviewer(review_path).review(references, findings) + + assert len(decisions) == 1 + assert decisions[0].state == "accepted" + assert decisions[0].page_compatible is True + assert decisions[0].provenance.reviewer == "reviewer@example.test" diff --git a/tests/test_normalization.py b/tests/test_normalization.py new file mode 100644 index 0000000..4cc6382 --- /dev/null +++ b/tests/test_normalization.py @@ -0,0 +1,102 @@ +import json + +import pytest + +from processing_scripts.llm.registry import PROMPT_REGISTRY +from vision_aid.evaluation.normalization import ( + normalize_programmatic_findings, + normalize_prompt_response, +) + + +@pytest.mark.parametrize( + "prompt_name,output_type", + [(spec.name, spec.output_type) for spec in PROMPT_REGISTRY if not spec.is_summary], +) +def test_every_active_prompt_has_a_lossless_normalizer( + prompt_name: str, + output_type: str, +) -> None: + item = { + "problem": "Synthetic accessibility failure", + "location": "hero image", + "element": "img.hero", + "wcag": ["1.1.1"], + "recommended_fix": "Synthetic fix retained as evidence", + } + response = item if output_type == "object" else [item] + + result = normalize_prompt_response( + prompt_name, + json.dumps(response), + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + + assert result.parse_status == "parsed" + assert len(result.findings) == 1 + finding = result.findings[0] + assert finding.prompt == prompt_name + assert finding.raw_source == item + assert finding.problem == "Synthetic accessibility failure" + assert finding.location == "hero image" + assert finding.wcag_evidence == ("1.1.1",) + + +def test_normalization_records_fenced_empty_malformed_and_unknown_responses() -> None: + fenced = normalize_prompt_response( + "link_clarity", + '```json\n[{"issue": "Unclear link", "text": "Read more"}]\n```', + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + empty = normalize_prompt_response( + "link_clarity", + "[]", + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + malformed = normalize_prompt_response( + "link_clarity", + "successful but not JSON", + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + unknown = normalize_prompt_response( + "invented_prompt", + "[]", + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + + assert fenced.parse_status == "parsed" + assert empty.parse_status == "empty" + assert malformed.parse_status == "malformed" + assert malformed.findings == () + assert malformed.raw_response == "successful but not JSON" + assert unknown.parse_status == "unknown_prompt" + + +def test_programmatic_normalization_retains_rule_and_raw_source() -> None: + raw = { + "rule_id": "HEAD_002", + "message": "Multiple H1 elements", + "element": "

Second

", + "location": "line 20", + "wcag": "1.3.1", + } + + findings = normalize_programmatic_findings( + [raw], + run_id="run-1", + page_url="https://example.test/", + ) + + assert findings[0].source == "programmatic" + assert findings[0].problem_family == "HEAD_002" + assert findings[0].raw_source == raw diff --git a/tests/test_programmatic_evaluation.py b/tests/test_programmatic_evaluation.py new file mode 100644 index 0000000..e05a4c3 --- /dev/null +++ b/tests/test_programmatic_evaluation.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from vision_aid.evaluation.programmatic import run_programmatic_audit + + +def test_existing_programmatic_checks_are_adapted_without_duplicating_rules( + tmp_path: Path, +) -> None: + html = tmp_path / "benchmark.html" + html.write_text( + "Synthetic" + "

One

Two

", + encoding="utf-8", + ) + + result = run_programmatic_audit( + html, + run_id="run-1", + page_url="https://example.test/", + ) + + rule_ids = {finding.problem_family for finding in result.findings} + assert "HEAD_002" in rule_ids + assert "NON_TEXT_001" in rule_ids + assert result.raw_findings + assert all(finding.raw_source in result.raw_findings for finding in result.findings) diff --git a/tests/test_reference_set.py b/tests/test_reference_set.py new file mode 100644 index 0000000..af3909c --- /dev/null +++ b/tests/test_reference_set.py @@ -0,0 +1,133 @@ +import hashlib +from pathlib import Path + +import openpyxl +import pytest + +from vision_aid.evaluation.references import ( + HumanEligibilityReviewer, + ReferenceSetError, + export_eligibility_workbook, + import_homepage_references, +) + + +def _write_workbook(path: Path) -> str: + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Defects" + sheet.append(["Page", "URL", "Problem", "Location", "WCAG", "Recommendation"]) + sheet.append( + [ + "Home", + "https://example.test/", + "Decorative image has descriptive alt text\n(second line retained)", + "Hero image", + "1.1.1", + "Use empty alternative text", + ] + ) + sheet.append( + [ + "Global", + "All pages", + "Heading labels are unclear", + "Primary navigation", + "2.4.6", + "Describe the section", + ] + ) + sheet.append( + [ + "Contact", + "https://example.test/contact", + "Contact-only issue", + "Form", + "3.3.2", + "Add instructions", + ] + ) + workbook.save(path) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_import_homepage_references_retains_every_home_and_global_source_row( + tmp_path: Path, +) -> None: + workbook_path = tmp_path / "synthetic.xlsx" + checksum = _write_workbook(workbook_path) + + reference_set = import_homepage_references( + workbook_path, + expected_sha256=checksum, + homepage_url="https://example.test/", + version="synthetic-v1", + ) + + assert [ + (item.source_sheet, item.source_row) for item in reference_set.references + ] == [ + ("Defects", 2), + ("Defects", 3), + ] + assert reference_set.references[0].raw_evidence["Problem"] == ( + "Decorative image has descriptive alt text\n(second line retained)" + ) + assert reference_set.references[1].page_scope == "global" + assert reference_set.references[1].canonical_url == "https://example.test/" + assert reference_set.workbook_sha256 == checksum + + with pytest.raises(ReferenceSetError, match="checksum drift"): + import_homepage_references( + workbook_path, + expected_sha256="0" * 64, + homepage_url="https://example.test/", + version="synthetic-v1", + ) + + +def test_human_eligibility_review_round_trips_validated_provenance( + tmp_path: Path, +) -> None: + workbook_path = tmp_path / "synthetic.xlsx" + checksum = _write_workbook(workbook_path) + reference_set = import_homepage_references( + workbook_path, + expected_sha256=checksum, + homepage_url="https://example.test/", + version="synthetic-v1", + ) + review_path = tmp_path / "eligibility.xlsx" + export_eligibility_workbook(reference_set, review_path) + + review = openpyxl.load_workbook(review_path) + sheet = review["Eligibility"] + headings = {cell.value: cell.column for cell in sheet[1]} + decisions = [ + ("llm_eligible", "accepted", "Human judgment is required."), + ("programmatic", "accepted", "A deterministic rule owns this defect."), + ] + for row, (classification, state, rationale) in enumerate(decisions, start=2): + sheet.cell(row, headings["classification"], classification) + sheet.cell(row, headings["decision"], state) + sheet.cell(row, headings["rationale"], rationale) + sheet.cell(row, headings["reviewer"], "reviewer@example.test") + sheet.cell(row, headings["confidence"], 0.9) + sheet.cell(row, headings["timestamp"], "2026-09-19T12:00:00Z") + review.save(review_path) + + reviewed = HumanEligibilityReviewer(review_path).review(reference_set) + + assert [item.eligibility for item in reviewed.references] == [ + "llm_eligible", + "programmatic", + ] + assert reviewed.references[0].eligibility_review is not None + assert reviewed.references[0].eligibility_review.reviewer == ( + "reviewer@example.test" + ) + + sheet.cell(2, headings["classification"], "secret_fifth_state") + review.save(review_path) + with pytest.raises(ReferenceSetError, match="classification"): + HumanEligibilityReviewer(review_path).review(reference_set) diff --git a/tests/test_scoring_and_reporting.py b/tests/test_scoring_and_reporting.py new file mode 100644 index 0000000..2cd9e34 --- /dev/null +++ b/tests/test_scoring_and_reporting.py @@ -0,0 +1,279 @@ +import csv +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from vision_aid.evaluation.matching import MatchValidationError +from vision_aid.evaluation.reporting import write_reports +from vision_aid.evaluation.schemas import ( + CanonicalFinding, + MatchDecision, + ReferenceDefect, + ReferenceSet, + ReviewProvenance, + RunMetadata, + Usage, +) +from vision_aid.evaluation.scoring import rank_scores, score_evaluation + +REVIEW = ReviewProvenance( + reviewer="reviewer@example.test", + rationale="Evidence describes the same failure at the same location.", + confidence=0.95, + timestamp="2026-09-19T12:00:00Z", +) + + +def _reference( + identifier: str, + eligibility: str, + *, + required_subdefects: tuple[str, ...] = (), +) -> ReferenceDefect: + return ReferenceDefect( + reference_id=identifier, + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + source_sheet="Defects", + source_row=int(identifier[-1]), + raw_evidence={"Problem": identifier}, + canonical_url="https://example.test/", + page_scope="home", + problem=identifier, + location="hero", + wcag_evidence=("1.1.1",), + required_subdefects=required_subdefects, + eligibility=eligibility, # type: ignore[arg-type] + eligibility_state="accepted", + eligibility_review=REVIEW, + ) + + +def _finding(identifier: str, source: str = "audit") -> CanonicalFinding: + return CanonicalFinding( + finding_id=identifier, + source=source, # type: ignore[arg-type] + run_id="run-1", + model="gpt-5.6-luna" if source == "audit" else None, + prompt="decorative_verification", + checklist="CL03", + page_url="https://example.test/", + problem_family="nontext", + problem=identifier, + element="img", + location="hero", + wcag_evidence=("1.1.1",), + raw_source={"problem": identifier}, + parse_status="parsed", + ) + + +def _match(reference: str, finding: str, subdefect: str | None = None) -> MatchDecision: + return MatchDecision( + reference_id=reference, + finding_id=finding, + state="accepted", + provenance=REVIEW, + required_subdefect=subdefect, + page_compatible=True, + failure_compatible=True, + location_compatible=True, + ) + + +def test_deterministic_score_separates_model_programmatic_and_combined_coverage() -> ( + None +): + references = ReferenceSet( + version="v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=( + _reference("ref1", "llm_eligible"), + _reference("ref2", "llm_eligible", required_subdefects=("a", "b")), + _reference("ref3", "programmatic"), + _reference("ref4", "unavailable_evidence"), + _reference("ref5", "ambiguous"), + ), + ) + audit_findings = (_finding("finding1"), _finding("finding2"), _finding("extra")) + programmatic_findings = (_finding("program1", "programmatic"),) + run = RunMetadata( + run_id="run-1", + model="gpt-5.6-luna", + complete=True, + comparable_identity="benchmark-v1", + audit_cost_usd="0.001234", + evaluation_cost_usd="0.02", + usage=Usage(input_tokens=100, output_tokens=25, reasoning_tokens=5), + wall_time_seconds=4.5, + request_durations_seconds=(1.0, 2.0), + ) + + partial = score_evaluation( + references, + audit_findings, + programmatic_findings, + (_match("ref1", "finding1"), _match("ref2", "finding2", "a")), + (_match("ref3", "program1"),), + run, + parse_failures=("heading_structure",), + ) + complete = score_evaluation( + references, + audit_findings, + programmatic_findings, + ( + _match("ref1", "finding1"), + _match("ref2", "finding2", "a"), + _match("ref2", "extra", "b"), + ), + (_match("ref3", "program1"),), + run, + parse_failures=("heading_structure",), + ) + + assert partial.workbook_row_recall == {"caught": 1, "denominator": 2, "rate": "0.5"} + assert complete.workbook_row_recall == {"caught": 2, "denominator": 2, "rate": "1"} + assert complete.programmatic_coverage == { + "caught": 1, + "denominator": 1, + "rate": "1", + } + assert complete.combined_workbook_coverage == { + "caught": 3, + "denominator": 5, + "rate": "0.6", + } + assert complete.unmatched_audit_finding_ids == () + assert complete.parse_failures == ("heading_structure",) + assert complete.total_experiment_cost_usd == "0.021234" + assert ( + complete.to_dict() + == score_evaluation( + references, + audit_findings, + programmatic_findings, + ( + _match("ref1", "finding1"), + _match("ref2", "finding2", "a"), + _match("ref2", "extra", "b"), + ), + (_match("ref3", "program1"),), + run, + parse_failures=("heading_structure",), + ).to_dict() + ) + + +def test_match_conflicts_and_incomplete_runs_cannot_be_ranked() -> None: + references = ReferenceSet( + version="v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=( + _reference("ref1", "llm_eligible"), + _reference("ref2", "llm_eligible"), + ), + ) + finding = _finding("finding1") + run = RunMetadata( + run_id="run-1", + model="gpt-5.6-luna", + complete=False, + comparable_identity="benchmark-v1", + audit_cost_usd="0.01", + ) + with pytest.raises(MatchValidationError, match="one-to-one"): + score_evaluation( + references, + (finding,), + (), + (_match("ref1", "finding1"), _match("ref2", "finding1")), + (), + run, + ) + + score = score_evaluation(references, (finding,), (), (), (), run) + with pytest.raises(ValueError, match="incomplete"): + rank_scores((score,)) + + +def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: + references = ReferenceSet( + version="v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=(_reference("ref1", "llm_eligible"),), + ) + finding = _finding("finding1") + run = RunMetadata( + run_id="run-1", + model="gpt-5.6-luna", + complete=True, + comparable_identity="benchmark-v1", + audit_cost_usd="0.001", + ) + score = score_evaluation( + references, + (finding,), + (), + (_match("ref1", "finding1"),), + (), + run, + ) + + paths = write_reports(score, tmp_path) + + json_report = json.loads(paths.json.read_text(encoding="utf-8")) + with paths.csv.open(newline="", encoding="utf-8") as stream: + csv_rows = list(csv.DictReader(stream)) + markdown = paths.markdown.read_text(encoding="utf-8") + assert json_report["workbook_row_recall"]["caught"] == 1 + assert csv_rows[0]["model_caught"] == "True" + assert "1/1 (100.00%)" in markdown + + +def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: + references = ReferenceSet( + version="v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=(_reference("ref1", "llm_eligible"),), + ) + finding = _finding("finding1") + base_run = RunMetadata( + run_id="expensive", + model="model-expensive", + complete=True, + comparable_identity="benchmark-v1", + audit_cost_usd="0.0012349", + ) + expensive = score_evaluation( + references, (finding,), (), (_match("ref1", "finding1"),), (), base_run + ) + cheap = score_evaluation( + references, + (replace(finding, run_id="cheap"),), + (), + (_match("ref1", "finding1"),), + (), + replace( + base_run, + run_id="cheap", + model="model-cheap", + audit_cost_usd="0.0012341", + wall_time_seconds=999, + ), + ) + + assert [score.model for score in rank_scores((expensive, cheap))] == [ + "model-cheap", + "model-expensive", + ] diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py new file mode 100644 index 0000000..be81115 --- /dev/null +++ b/vision_aid/evaluation/audit.py @@ -0,0 +1,411 @@ +"""Production-pipeline adapter and guarded evaluation audit execution.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import time +from collections.abc import Callable +from dataclasses import asdict +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, Protocol + +from processing_scripts.llm_client.audit import AuditRequestClient, AuditRequestConfig +from vision_aid.evaluation.normalization import ( + normalize_programmatic_findings, + normalize_prompt_response, +) +from vision_aid.evaluation.pricing import PriceSchedule + +MODEL = "gpt-5.6-luna" +REQUEST_CONFIG = AuditRequestConfig( + model=MODEL, + reasoning_effort="medium", + temperature=None, + max_output_tokens=8192, +) +RETRYABLE_FAILURES = ("timeout", "rate_limit", "server_error") +BACKOFF_SECONDS = (1, 2) +RETRY_POLICY = { + "maximum_additional_attempts": 2, + "backoff_seconds": list(BACKOFF_SECONDS), + "retryable": list(RETRYABLE_FAILURES), +} + + +class AuditExecutionError(ValueError): + """An audit gate, identity check, or execution policy failed.""" + + +class RequestClient(Protocol): + def call(self, prompt: str) -> dict[str, Any]: + """Call one prompt and return the production client result shape.""" + + +Pipeline = Callable[..., dict[str, Any]] + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _json_digest(value: object) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _repository_state(repository: Path) -> dict[str, Any]: + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repository, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + status = subprocess.run( + ["git", "status", "--porcelain"], + cwd=repository, + capture_output=True, + text=True, + check=True, + ).stdout + return {"commit": commit, "dirty": bool(status.strip())} + except (OSError, subprocess.CalledProcessError): + return {"commit": None, "dirty": None} + + +def build_audit_plan( + snapshot_path: Path, + output_directory: Path, + *, + workbook_sha256: str, + snapshot_sha256: str, + reference_set_version: str, + eligibility_identity: str, + homepage_url: str = "benchmark-homepage", + pipeline: Pipeline | None = None, + repository: Path | None = None, + schedule: PriceSchedule | None = None, +) -> dict[str, Any]: + """Run the existing pipeline dry and freeze an authorization summary.""" + if _sha256(snapshot_path) != snapshot_sha256: + raise AuditExecutionError( + "Benchmark snapshot checksum drift detected before audit planning." + ) + if pipeline is None: + from entry_points.run_pipeline import run_pipeline + + pipeline = run_pipeline + output_directory.mkdir(parents=True, exist_ok=True) + production_manifest = pipeline( + html_path=str(snapshot_path), + output_dir=output_directory, + api_key=None, + model=MODEL, + dry_run=True, + include_summaries=False, + request_config=REQUEST_CONFIG, + ) + prompt_entries = production_manifest.get("prompts_dry_run", []) + prompts = [] + prompt_hashes: dict[str, str] = {} + for entry in prompt_entries: + name = str(entry["name"]) + prompt_path = output_directory / "prompts" / f"{name}.json" + prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) + prompt_text = str(prompt_data["prompt_text"]) + prompt_hashes[name] = hashlib.sha256(prompt_text.encode()).hexdigest() + prompts.append( + { + "name": name, + "checklist": entry["checklist"], + "wcag_criteria": entry.get("wcag_criteria", []), + "estimated_input_tokens": int(entry["prompt_tokens_est"]), + "path": str(prompt_path.relative_to(output_directory)), + } + ) + price_schedule = schedule or PriceSchedule.default() + parser_path = Path(__file__).with_name("normalization.py") + now = datetime.now(UTC).isoformat().replace("+00:00", "Z") + run_id = "dry-" + _json_digest([snapshot_sha256, now])[:16] + raw_programmatic_path = output_directory / "programmatic_findings.json" + raw_programmatic = json.loads(raw_programmatic_path.read_text(encoding="utf-8")) + canonical_programmatic = normalize_programmatic_findings( + raw_programmatic, + run_id=run_id, + page_url=homepage_url, + ) + canonical_programmatic_path = output_directory / "canonical-programmatic.json" + canonical_programmatic_path.write_text( + json.dumps( + [asdict(item) for item in canonical_programmatic], + indent=2, + sort_keys=True, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + plan: dict[str, Any] = { + "kind": "model-evaluation-audit-plan", + "created_at": now, + "run_id": run_id, + "homepage_url": homepage_url, + "workbook_sha256": workbook_sha256, + "snapshot_sha256": snapshot_sha256, + "reference_set_version": reference_set_version, + "eligibility_identity": eligibility_identity, + "configuration": { + "model": MODEL, + "reasoning_effort": "medium", + "endpoint": "chat.completions", + "temperature": None, + "max_output_tokens": 8192, + "include_summaries": False, + "execution": "sequential", + }, + "filters": production_manifest.get("pass1_filters_active", []), + "skipped_prompts": production_manifest.get("prompts_skipped", []), + "request_count": len(prompts), + "estimated_input_tokens": sum( + item["estimated_input_tokens"] for item in prompts + ), + "prompts": prompts, + "prompt_hashes": prompt_hashes, + "parser_hash": _sha256(parser_path), + "pricing_version": price_schedule.version, + "pricing_identity": price_schedule.identity, + "retry_policy": RETRY_POLICY, + "repository": _repository_state(repository or Path.cwd()), + "programmatic_findings": { + "count": len(canonical_programmatic), + "path": str(canonical_programmatic_path.relative_to(output_directory)), + "sha256": _sha256(canonical_programmatic_path), + }, + } + plan["benchmark_identity"] = _json_digest( + { + key: plan[key] + for key in ( + "workbook_sha256", + "snapshot_sha256", + "reference_set_version", + "eligibility_identity", + "configuration", + "filters", + "prompt_hashes", + "parser_hash", + "pricing_identity", + ) + } + ) + plan["authorization_digest"] = _json_digest(plan) + (output_directory / "evaluation-manifest.json").write_text( + json.dumps(plan, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return plan + + +def _failure_classification(result: dict[str, Any]) -> str: + failure = result.get("failure") or {} + status = failure.get("status_code") + failure_type = str(failure.get("type") or "").casefold() + if status == 429 or "ratelimit" in failure_type: + return "rate_limit" + if ( + status in {408, 409} + or "timeout" in failure_type + or "connection" in failure_type + ): + return "timeout" + if isinstance(status, int) and status >= 500: + return "server_error" + if status in {401, 403} or "auth" in failure_type or "permission" in failure_type: + return "authentication" + if isinstance(status, int) and 400 <= status < 500: + return "invalid_request" + return "permanent" + + +def _empty_usage() -> dict[str, int]: + return { + "input_tokens": 0, + "cached_input_tokens": 0, + "cache_creation_input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + } + + +def _add_usage(total: dict[str, int], result: dict[str, Any]) -> None: + usage = result.get("usage") or {} + for key in total: + total[key] += int(usage.get(key, 0) or 0) + + +def execute_authorized_audit( + plan_path: Path, + *, + live: bool = False, + authorization: str | None, + api_key: str | None, + max_audit_cost_usd: str | None, + client: RequestClient | None = None, + schedule: PriceSchedule | None = None, + sleep: Callable[[float], None] = time.sleep, +) -> dict[str, Any]: + """Execute the exact reviewed plan only after every spending gate passes.""" + if not live: + raise AuditExecutionError("Live execution requires the explicit --live flag.") + if not api_key: + raise AuditExecutionError("Live execution requires OPENAI_API_KEY.") + if max_audit_cost_usd is None: + raise AuditExecutionError( + "Live execution requires --max-audit-cost-usd as a spending guardrail." + ) + try: + cost_limit = Decimal(max_audit_cost_usd) + except Exception as error: + raise AuditExecutionError( + "The audit cost limit must be a decimal amount." + ) from error + if cost_limit <= 0: + raise AuditExecutionError("The audit cost limit must be greater than zero.") + plan = json.loads(plan_path.read_text(encoding="utf-8")) + recorded_digest = plan.get("authorization_digest") + digest_input = dict(plan) + digest_input.pop("authorization_digest", None) + if recorded_digest != _json_digest(digest_input): + raise AuditExecutionError( + "The saved dry-run summary has changed since creation." + ) + if authorization != recorded_digest: + raise AuditExecutionError( + "Live authorization does not match the exact summarized configuration." + ) + if plan.get("configuration") != { + "model": MODEL, + "reasoning_effort": "medium", + "endpoint": "chat.completions", + "temperature": None, + "max_output_tokens": 8192, + "include_summaries": False, + "execution": "sequential", + }: + raise AuditExecutionError( + "The dry-run configuration is not the frozen Luna POC." + ) + price_schedule = schedule or PriceSchedule.default() + if plan.get("pricing_identity") != price_schedule.identity: + raise AuditExecutionError( + "The price schedule differs from the authorized plan." + ) + request_client = client or AuditRequestClient( + api_key=api_key, + request_config=REQUEST_CONFIG, + ) + output_directory = plan_path.parent + requests: list[dict[str, Any]] = [] + totals = _empty_usage() + format_failures: list[str] = [] + canonical_findings: list[dict[str, Any]] = [] + complete = True + incomplete_reason: str | None = None + started = time.monotonic() + for prompt in plan["prompts"]: + current_cost = Decimal(price_schedule.estimate(MODEL, totals)) + if current_cost >= cost_limit: + complete = False + incomplete_reason = "audit_cost_limit_exhausted" + break + prompt_path = output_directory / prompt["path"] + prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) + prompt_text = str(prompt_data["prompt_text"]) + name = str(prompt["name"]) + if ( + hashlib.sha256(prompt_text.encode()).hexdigest() + != plan["prompt_hashes"][name] + ): + raise AuditExecutionError(f"Prompt content drift detected for {name}.") + attempts = [] + final_result: dict[str, Any] | None = None + for attempt_index in range(3): + result = request_client.call(prompt_text) + _add_usage(totals, result) + classification = ( + "success" if result.get("success") else _failure_classification(result) + ) + attempts.append( + { + "attempt": attempt_index + 1, + "classification": classification, + "result": result, + } + ) + final_result = result + if result.get("success"): + break + if classification not in RETRYABLE_FAILURES: + complete = False + incomplete_reason = classification + break + if attempt_index < 2: + sleep(BACKOFF_SECONDS[attempt_index]) + assert final_result is not None + request_record = {"name": name, "attempts": attempts} + requests.append(request_record) + if not final_result.get("success"): + complete = False + incomplete_reason = incomplete_reason or "transient_failures_exhausted" + break + normalized = normalize_prompt_response( + name, + str(final_result.get("response") or ""), + run_id=str(plan["run_id"]), + model=MODEL, + page_url=str(plan["homepage_url"]), + ) + if normalized.parse_status == "malformed": + format_failures.append(name) + canonical_findings.extend(asdict(item) for item in normalized.findings) + manifest = { + **plan, + "kind": "model-evaluation-live-run", + "authorized_plan_digest": recorded_digest, + "maximum_audit_cost_usd": str(cost_limit), + "completed_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "complete": complete, + "rankable": complete, + "incomplete_reason": incomplete_reason, + "requests": requests, + "usage": totals, + "estimated_audit_cost_usd": price_schedule.estimate(MODEL, totals), + "wall_time_seconds": round(time.monotonic() - started, 6), + "request_durations_seconds": [ + attempt["result"].get("duration_seconds", 0) + for request in requests + for attempt in request["attempts"] + ], + "format_failures": format_failures, + "canonical_findings": canonical_findings, + } + (output_directory / "canonical-audit-findings.json").write_text( + json.dumps(canonical_findings, indent=2, sort_keys=True, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + (output_directory / "live-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 6c6dd7e..a47b6ae 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -1,9 +1,33 @@ """Command-line shell for the private model-evaluation lifecycle.""" import argparse +import hashlib +import json +import os from collections.abc import Callable, Sequence from pathlib import Path +from vision_aid.evaluation.matching import ( + HumanMatchReviewer, + MatchValidationError, + export_match_workbook, +) +from vision_aid.evaluation.references import ( + HumanEligibilityReviewer, + ReferenceSetError, + export_eligibility_workbook, + import_homepage_references, +) +from vision_aid.evaluation.reporting import write_reports +from vision_aid.evaluation.scoring import score_evaluation +from vision_aid.evaluation.serialization import ( + load_findings, + load_match_decisions, + load_reference_set, + load_run_metadata, + save_match_decisions, + save_reference_set, +) from vision_aid.evaluation.snapshot import prepare_benchmark_snapshot from vision_aid.evaluation.workspace import ( DEFAULT_WORKBOOK_FILENAME, @@ -42,6 +66,19 @@ def build_parser() -> argparse.ArgumentParser: f"authorized local source workbook (default: ./{DEFAULT_WORKBOOK_FILENAME})" ), ) + prepare.add_argument( + "--workbook-sha256", + help="authorized SHA-256 required to import homepage reference rows", + ) + prepare.add_argument( + "--homepage-url", + help="canonical Pristine homepage URL used to select Home and Global rows", + ) + prepare.add_argument( + "--reference-set-version", + default="pristine-homepage-v1", + help="curator-assigned version for the imported reference set", + ) prepare.add_argument( "--snapshot-url", help="HTTP(S) source URL for an immutable Pristine homepage capture", @@ -59,18 +96,49 @@ def build_parser() -> argparse.ArgumentParser: "audit", help="inspect the prepared audit plan in network-free dry-run mode", ) + audit.add_argument("--live", action="store_true", help="enable billable execution") + audit.add_argument("--plan", type=Path, help="exact dry-run manifest to execute") + audit.add_argument("--authorize", help="authorization digest shown by dry run") + audit.add_argument("--max-audit-cost-usd", help="required live cost guardrail") + audit.add_argument("--reference-set", type=Path, help="approved reference-set JSON") + audit.add_argument( + "--run-dir", type=Path, help="private output directory for the plan" + ) audit.set_defaults(handler=_audit) review = workflows.add_parser( "review", help="work with private eligibility and match review artifacts", ) + review.add_argument( + "--eligibility-workbook", + type=Path, + help="apply a completed Eligibility workbook", + ) + review.add_argument("--reference-set", type=Path, help="reference-set JSON input") + review.add_argument("--findings", type=Path, help="canonical findings JSON") + review.add_argument( + "--matches-workbook", + type=Path, + help="generate or apply a Matches workbook", + ) + review.add_argument( + "--import-matches", + action="store_true", + help="import decisions from --matches-workbook instead of generating it", + ) review.set_defaults(handler=_review) report = workflows.add_parser( "report", help="generate private evaluation reports", ) + report.add_argument( + "--bundle", + type=Path, + help="private JSON bundle containing score inputs", + ) + report.add_argument("--output-dir", type=Path, help="private report directory") report.set_defaults(handler=_report) return parser @@ -82,10 +150,25 @@ def _prepare(arguments: argparse.Namespace) -> int: workspace.initialize() print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") print(f"Private source workbook: {workbook}") - if arguments.temporal_assumption and not arguments.snapshot_url: + if bool(arguments.workbook_sha256) != bool(arguments.homepage_url): raise PrivateInputError( - "--temporal-assumption requires --snapshot-url." + "Reference import requires both --workbook-sha256 and --homepage-url." ) + if arguments.workbook_sha256: + reference_set = import_homepage_references( + workbook, + expected_sha256=arguments.workbook_sha256, + homepage_url=arguments.homepage_url, + version=arguments.reference_set_version, + ) + imported_path = workspace.root / "references" / "imported.json" + review_path = workspace.root / "reviews" / "eligibility.xlsx" + save_reference_set(imported_path, reference_set) + export_eligibility_workbook(reference_set, review_path) + print(f"Imported reference set: {imported_path.resolve()}") + print(f"Eligibility review workbook: {review_path.resolve()}") + if arguments.temporal_assumption and not arguments.snapshot_url: + raise PrivateInputError("--temporal-assumption requires --snapshot-url.") if arguments.snapshot_url: snapshot = prepare_benchmark_snapshot( workspace, @@ -99,9 +182,57 @@ def _prepare(arguments: argparse.Namespace) -> int: def _audit(arguments: argparse.Namespace) -> int: - """Expose the safe audit shell without enabling live execution yet.""" + """Build a no-cost plan or execute its exact authorized configuration.""" workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() + if arguments.live: + from vision_aid.evaluation.audit import execute_authorized_audit + + if arguments.plan is None: + raise PrivateInputError("--live requires --plan from a reviewed dry run.") + manifest = execute_authorized_audit( + arguments.plan, + live=True, + authorization=arguments.authorize, + api_key=os.environ.get("OPENAI_API_KEY"), + max_audit_cost_usd=arguments.max_audit_cost_usd, + ) + print(f"Live run complete: {manifest['complete']}") + print(f"Estimated audit cost: ${manifest['estimated_audit_cost_usd']}") + return 0 + reference_path = arguments.reference_set or ( + workspace.root / "references" / "approved.json" + ) + snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" + snapshot_path = snapshot_directory / "source.html" + metadata_path = snapshot_directory / "metadata.json" + if reference_path.is_file() and snapshot_path.is_file() and metadata_path.is_file(): + from vision_aid.evaluation.audit import build_audit_plan + + reference_set = load_reference_set(reference_path) + if any( + item.eligibility_state != "accepted" for item in reference_set.references + ): + raise PrivateInputError( + "The reference set has unresolved eligibility decisions." + ) + snapshot_metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + eligibility_identity = hashlib.sha256(reference_path.read_bytes()).hexdigest() + run_directory = arguments.run_dir or workspace.root / "runs" / "dry-run" + plan = build_audit_plan( + snapshot_path, + run_directory, + workbook_sha256=reference_set.workbook_sha256, + snapshot_sha256=snapshot_metadata["sha256"], + reference_set_version=reference_set.version, + eligibility_identity=eligibility_identity, + homepage_url=reference_set.homepage_url, + ) + print(f"DRY RUN: {plan['request_count']} sequential requests planned.") + print(f"Estimated input tokens: {plan['estimated_input_tokens']}") + print(f"Authorization digest: {plan['authorization_digest']}") + print(f"Plan: {(run_directory / 'evaluation-manifest.json').resolve()}") + return 0 print("DRY RUN: No network requests were made and no API usage was incurred.") print(f"Private evaluation workspace: {workspace.root.resolve()}") return 0 @@ -111,6 +242,36 @@ def _review(arguments: argparse.Namespace) -> int: """Expose the private review workflow shell.""" workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() + if arguments.eligibility_workbook: + reference_path = arguments.reference_set or ( + workspace.root / "references" / "imported.json" + ) + reviewed = HumanEligibilityReviewer(arguments.eligibility_workbook).review( + load_reference_set(reference_path) + ) + approved_path = workspace.root / "references" / "approved.json" + save_reference_set(approved_path, reviewed) + print(f"Validated reference set: {approved_path.resolve()}") + return 0 + if arguments.matches_workbook: + if not arguments.reference_set or not arguments.findings: + raise PrivateInputError( + "Match review requires --reference-set and --findings." + ) + references = load_reference_set(arguments.reference_set).references + findings = load_findings(arguments.findings) + if arguments.import_matches: + decisions = HumanMatchReviewer(arguments.matches_workbook).review( + references, + findings, + ) + output = workspace.root / "reviews" / "match-decisions.json" + save_match_decisions(output, decisions) + print(f"Validated match decisions: {output.resolve()}") + else: + export_match_workbook(references, findings, arguments.matches_workbook) + print(f"Matches review workbook: {arguments.matches_workbook.resolve()}") + return 0 print(f"Private review workspace: {(workspace.root / 'reviews').resolve()}") return 0 @@ -119,6 +280,24 @@ def _report(arguments: argparse.Namespace) -> int: """Expose the private report workflow shell.""" workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() + if arguments.bundle: + bundle = json.loads(arguments.bundle.read_text(encoding="utf-8")) + base = arguments.bundle.parent + score = score_evaluation( + load_reference_set(base / bundle["reference_set"]), + load_findings(base / bundle["audit_findings"]), + load_findings(base / bundle["programmatic_findings"]), + load_match_decisions(base / bundle["audit_matches"]), + load_match_decisions(base / bundle["programmatic_matches"]), + load_run_metadata(bundle["run"]), + parse_failures=tuple(bundle.get("parse_failures", ())), + ) + output_directory = arguments.output_dir or workspace.root / "reports" + paths = write_reports(score, output_directory) + print(f"JSON report: {paths.json.resolve()}") + print(f"CSV report: {paths.csv.resolve()}") + print(f"Markdown report: {paths.markdown.resolve()}") + return 0 print(f"Private report workspace: {(workspace.root / 'reports').resolve()}") return 0 @@ -130,7 +309,13 @@ def main(argv: Sequence[str] | None = None) -> int: handler: CommandHandler = arguments.handler try: return handler(arguments) - except PrivateInputError as error: + except ( + MatchValidationError, + PrivateInputError, + ReferenceSetError, + OSError, + ValueError, + ) as error: parser.error(str(error)) diff --git a/vision_aid/evaluation/matching.py b/vision_aid/evaluation/matching.py new file mode 100644 index 0000000..bc40673 --- /dev/null +++ b/vision_aid/evaluation/matching.py @@ -0,0 +1,369 @@ +"""Candidate generation and reviewed one-to-one match validation.""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Sequence +from pathlib import Path +from typing import Any, Protocol + +import openpyxl +from openpyxl.worksheet.datavalidation import DataValidation + +from vision_aid.evaluation.schemas import ( + CanonicalFinding, + MatchDecision, + ReferenceDefect, + ReviewProvenance, +) + + +class MatchValidationError(ValueError): + """Match decisions are unresolved, incompatible, or conflicting.""" + + +class MatchReviewer(Protocol): + """Adapter contract for human or future model match review.""" + + def review( + self, + references: Sequence[ReferenceDefect], + findings: Sequence[CanonicalFinding], + ) -> tuple[MatchDecision, ...]: + """Return validated decisions for candidate finding/reference pairs.""" + + +MATCH_COLUMNS = ( + "reference_id", + "finding_id", + "reference_scope", + "reference_problem", + "reference_location", + "reference_wcag", + "reference_raw_evidence", + "finding_prompt", + "finding_problem", + "finding_element", + "finding_location", + "finding_wcag", + "finding_raw_source", + "decision", + "page_compatible", + "failure_compatible", + "location_compatible", + "required_subdefect", + "rationale", + "reviewer", + "confidence", + "timestamp", + "notes", +) + + +def generate_candidates( + references: Sequence[ReferenceDefect], + findings: Sequence[CanonicalFinding], +) -> tuple[tuple[str, str], ...]: + """Remove only hard page/element incompatibilities; never award credit.""" + candidates: list[tuple[str, str]] = [] + for reference in references: + for finding in findings: + page_compatible = ( + reference.page_scope == "global" + or reference.canonical_url == finding.page_url + ) + if not page_compatible: + continue + reference_location = reference.location.casefold() + finding_evidence = f"{finding.element} {finding.location}".casefold() + if reference_location and finding_evidence: + # Only treat explicit, disjoint element categories as impossible. + categories = ("image", "heading", "link", "form", "table", "media") + ref_categories = { + item for item in categories if item in reference_location + } + finding_categories = { + item for item in categories if item in finding_evidence + } + if ( + ref_categories + and finding_categories + and ref_categories.isdisjoint(finding_categories) + ): + continue + candidates.append((reference.reference_id, finding.finding_id)) + return tuple(candidates) + + +def validate_match_decisions( + references: Sequence[ReferenceDefect], + findings: Sequence[CanonicalFinding], + decisions: Sequence[MatchDecision], + *, + require_resolved: bool = True, +) -> tuple[MatchDecision, ...]: + """Require resolved evidence-backed decisions and one-to-one findings.""" + reference_by_id = {item.reference_id: item for item in references} + finding_ids = {item.finding_id for item in findings} + pair_counts = Counter((item.reference_id, item.finding_id) for item in decisions) + repeated = [pair for pair, count in pair_counts.items() if count > 1] + if repeated: + raise MatchValidationError( + f"Match review repeats candidate pair {repeated[0]!r}." + ) + for decision in decisions: + if decision.reference_id not in reference_by_id: + raise MatchValidationError( + f"Unknown reference in match review: {decision.reference_id}." + ) + if decision.finding_id not in finding_ids: + raise MatchValidationError( + f"Unknown finding in match review: {decision.finding_id}." + ) + if require_resolved and decision.state == "needs_review": + raise MatchValidationError( + "Unresolved needs_review match decisions cannot be scored." + ) + if not decision.provenance.reviewer or not decision.provenance.rationale: + raise MatchValidationError("Every match decision requires provenance.") + if not 0 <= decision.provenance.confidence <= 1: + raise MatchValidationError("Match confidence must be 0 through 1.") + if decision.state != "accepted": + continue + if not all( + ( + decision.page_compatible, + decision.failure_compatible, + decision.location_compatible, + ) + ): + raise MatchValidationError( + "An accepted match requires compatible page, underlying failure, " + "and location or element-group evidence." + ) + reference = reference_by_id[decision.reference_id] + if reference.required_subdefects: + if decision.required_subdefect not in reference.required_subdefects: + raise MatchValidationError( + f"Accepted match for {reference.reference_id} must identify one " + "of its required sub-defects." + ) + elif decision.required_subdefect: + raise MatchValidationError( + f"Reference {reference.reference_id} has no required sub-defects." + ) + accepted = [item for item in decisions if item.state == "accepted"] + finding_counts = Counter(item.finding_id for item in accepted) + conflicts = [item for item, count in finding_counts.items() if count > 1] + if conflicts: + raise MatchValidationError( + "Accepted matches must be one-to-one; finding " + f"{conflicts[0]} claims multiple reference defects." + ) + plain_reference_counts = Counter( + item.reference_id + for item in accepted + if not reference_by_id[item.reference_id].required_subdefects + ) + conflicts = [item for item, count in plain_reference_counts.items() if count > 1] + if conflicts: + raise MatchValidationError( + "Accepted matches must be one-to-one; reference " + f"{conflicts[0]} is claimed by multiple findings." + ) + subdefect_counts = Counter( + (item.reference_id, item.required_subdefect) + for item in accepted + if item.required_subdefect is not None + ) + subdefect_conflicts = [ + item for item, count in subdefect_counts.items() if count > 1 + ] + if subdefect_conflicts: + raise MatchValidationError( + "Accepted matches must be one-to-one at the required sub-defect level." + ) + return tuple(decisions) + + +def export_match_workbook( + references: Sequence[ReferenceDefect], + findings: Sequence[CanonicalFinding], + output_path: Path, +) -> None: + """Create an evidence-rich private review workbook for plausible pairs.""" + reference_by_id = {item.reference_id: item for item in references} + finding_by_id = {item.finding_id: item for item in findings} + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Matches" + sheet.append(MATCH_COLUMNS) + for reference_id, finding_id in generate_candidates(references, findings): + reference = reference_by_id[reference_id] + finding = finding_by_id[finding_id] + sheet.append( + ( + reference_id, + finding_id, + reference.page_scope, + reference.problem, + reference.location, + ", ".join(reference.wcag_evidence), + json.dumps(reference.raw_evidence, ensure_ascii=False, sort_keys=True), + finding.prompt, + finding.problem, + finding.element, + finding.location, + ", ".join(finding.wcag_evidence), + json.dumps(finding.raw_source, ensure_ascii=False, sort_keys=True), + "needs_review", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ) + ) + sheet.freeze_panes = "A2" + sheet.auto_filter.ref = sheet.dimensions + for column in ("D", "E", "G", "I", "K", "M", "S", "W"): + sheet.column_dimensions[column].width = 40 + for row in sheet.iter_rows(min_row=2): + for cell in row: + cell.alignment = openpyxl.styles.Alignment( + wrap_text=True, + vertical="top", + ) + decisions = DataValidation( + type="list", + formula1='"accepted,rejected,needs_review"', + ) + booleans = DataValidation(type="list", formula1='"TRUE,FALSE"') + sheet.add_data_validation(decisions) + sheet.add_data_validation(booleans) + decisions.add(f"N2:N{max(2, sheet.max_row)}") + booleans.add(f"O2:Q{max(2, sheet.max_row)}") + output_path.parent.mkdir(parents=True, exist_ok=True) + workbook.save(output_path) + + +def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def _boolean(value: Any, *, row: int, field: str) -> bool | None: + if value in (None, ""): + return None + if isinstance(value, bool): + return value + normalized = str(value).strip().casefold() + if normalized in {"true", "yes", "1"}: + return True + if normalized in {"false", "no", "0"}: + return False + raise MatchValidationError(f"Match row {row} has invalid {field} value.") + + +class HumanMatchReviewer: + """Load human match decisions through the shared reviewer contract.""" + + def __init__(self, review_path: Path): + self.review_path = review_path + + def review( + self, + references: Sequence[ReferenceDefect], + findings: Sequence[CanonicalFinding], + ) -> tuple[MatchDecision, ...]: + """Load provenance-complete decisions and validate accepted conflicts.""" + try: + workbook = openpyxl.load_workbook( + self.review_path, + read_only=True, + data_only=True, + ) + sheet = workbook["Matches"] + except (OSError, KeyError, ValueError) as error: + raise MatchValidationError( + f"Could not read Matches review workbook: {error}" + ) from error + rows = sheet.iter_rows(values_only=True) + header_values = next(rows, None) or () + headers = [_text(value) for value in header_values] + missing = set(MATCH_COLUMNS) - set(headers) + if missing: + raise MatchValidationError( + "Matches review is missing columns: " + ", ".join(sorted(missing)) + ) + decisions = [] + for row_number, values in enumerate(rows, start=2): + record = dict(zip(headers, values, strict=False)) + reference_id = _text(record.get("reference_id")) + finding_id = _text(record.get("finding_id")) + if not reference_id and not finding_id: + continue + state = _text(record.get("decision")) + if state not in {"accepted", "rejected", "needs_review"}: + raise MatchValidationError( + f"Match row {row_number} has invalid decision {state!r}." + ) + reviewer = _text(record.get("reviewer")) + rationale = _text(record.get("rationale")) + timestamp = _text(record.get("timestamp")) + if not reviewer or not rationale or not timestamp: + raise MatchValidationError( + f"Match row {row_number} requires reviewer, rationale, and timestamp." + ) + try: + raw_confidence = record.get("confidence") + if raw_confidence is None: + raise ValueError + confidence = float(raw_confidence) + except (TypeError, ValueError) as error: + raise MatchValidationError( + f"Match row {row_number} requires numeric confidence." + ) from error + decisions.append( + MatchDecision( + reference_id=reference_id, + finding_id=finding_id, + state=state, # type: ignore[arg-type] + provenance=ReviewProvenance( + reviewer=reviewer, + rationale=rationale, + confidence=confidence, + timestamp=timestamp, + ), + required_subdefect=( + _text(record.get("required_subdefect")) or None + ), + page_compatible=_boolean( + record.get("page_compatible"), + row=row_number, + field="page_compatible", + ), + failure_compatible=_boolean( + record.get("failure_compatible"), + row=row_number, + field="failure_compatible", + ), + location_compatible=_boolean( + record.get("location_compatible"), + row=row_number, + field="location_compatible", + ), + notes=_text(record.get("notes")), + ) + ) + workbook.close() + return validate_match_decisions( + references, + findings, + decisions, + require_resolved=False, + ) diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py new file mode 100644 index 0000000..d9c4a86 --- /dev/null +++ b/vision_aid/evaluation/normalization.py @@ -0,0 +1,201 @@ +"""Lossless deterministic normalization of raw audit outcomes.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec +from vision_aid.evaluation.schemas import CanonicalFinding + +_PROMPTS = {item.name: item for item in PROMPT_REGISTRY if not item.is_summary} +_PROBLEM_KEYS = ( + "problem", + "issue", + "issue_title", + "description", + "finding", + "reason", + "actual_result", + "summary", +) +_LOCATION_KEYS = ( + "location", + "location_hint", + "selector", + "xpath", + "text", + "link_text", + "src", +) +_ELEMENT_KEYS = ("element", "element_name", "tag", "html", "selector") +_WCAG_KEYS = ("wcag", "wcag_sc", "wcag_criteria", "criterion", "criteria") + + +@dataclass(frozen=True) +class NormalizationResult: + """Findings plus the parse outcome for one successful raw response.""" + + prompt: str + parse_status: str + raw_response: str + findings: tuple[CanonicalFinding, ...] + error: str | None = None + + +def _strip_fence(value: str) -> str: + match = re.fullmatch(r"\s*```(?:json)?\s*(.*?)\s*```\s*", value, re.DOTALL) + return match.group(1) if match else value.strip() + + +def _first_text(item: dict[str, Any], keys: Iterable[str]) -> str: + for key in keys: + value = item.get(key) + if value is None or value == "": + continue + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + return "" + + +def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: + values: list[str] = [] + for key in _WCAG_KEYS: + value = item.get(key) + if isinstance(value, list): + values.extend(str(part) for part in value) + elif value: + values.append(str(value)) + criteria = re.findall(r"\b\d+\.\d+\.\d+\b", " ".join(values)) + return tuple(dict.fromkeys(criteria or defaults)) + + +def _items(parsed: Any, spec: PromptSpec) -> list[dict[str, Any]] | None: + if isinstance(parsed, list): + return [item for item in parsed if isinstance(item, dict)] + if not isinstance(parsed, dict): + return None + for key in ("findings", "issues", "problems", "violations", "results"): + nested = parsed.get(key) + if isinstance(nested, list): + return [item for item in nested if isinstance(item, dict)] + if spec.output_type == "object": + if not parsed: + return [] + return [parsed] + return None + + +def normalize_prompt_response( + prompt_name: str, + raw_response: str, + *, + run_id: str, + model: str, + page_url: str, +) -> NormalizationResult: + """Normalize one prompt directly, without suppression or deduplication.""" + spec = _PROMPTS.get(prompt_name) + if spec is None: + return NormalizationResult( + prompt=prompt_name, + parse_status="unknown_prompt", + raw_response=raw_response, + findings=(), + error=f"No fixed evaluation normalizer exists for {prompt_name!r}.", + ) + try: + parsed = json.loads(_strip_fence(raw_response)) + except (json.JSONDecodeError, TypeError) as error: + return NormalizationResult( + prompt=prompt_name, + parse_status="malformed", + raw_response=raw_response, + findings=(), + error=str(error), + ) + items = _items(parsed, spec) + if items is None: + return NormalizationResult( + prompt=prompt_name, + parse_status="malformed", + raw_response=raw_response, + findings=(), + error=f"Expected a JSON {spec.output_type} response.", + ) + findings = [] + for index, item in enumerate(items): + canonical_item = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + identity = f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() + findings.append( + CanonicalFinding( + finding_id="finding-" + hashlib.sha256(identity).hexdigest()[:20], + source="audit", + run_id=run_id, + model=model, + prompt=prompt_name, + checklist=spec.checklist, + page_url=page_url, + problem_family=prompt_name, + problem=_first_text(item, _PROBLEM_KEYS) or prompt_name, + element=_first_text(item, _ELEMENT_KEYS), + location=_first_text(item, _LOCATION_KEYS), + wcag_evidence=_wcag(item, spec.wcag_criteria), + raw_source=item, + parse_status="parsed", + ) + ) + return NormalizationResult( + prompt=prompt_name, + parse_status="parsed" if findings else "empty", + raw_response=raw_response, + findings=tuple(findings), + ) + + +def normalize_programmatic_findings( + raw_findings: Sequence[dict[str, Any]], + *, + run_id: str, + page_url: str, +) -> tuple[CanonicalFinding, ...]: + """Normalize checker output while retaining each complete source object.""" + normalized = [] + for index, item in enumerate(raw_findings): + canonical_item = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + identity = f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() + rule = str(item.get("rule_id") or item.get("rule") or "programmatic") + normalized.append( + CanonicalFinding( + finding_id="programmatic-" + hashlib.sha256(identity).hexdigest()[:20], + source="programmatic", + run_id=run_id, + model=None, + prompt=rule, + checklist=str(item.get("checklist") or "programmatic"), + page_url=page_url, + problem_family=rule, + problem=_first_text(item, _PROBLEM_KEYS + ("message",)) or rule, + element=_first_text(item, _ELEMENT_KEYS), + location=_first_text(item, _LOCATION_KEYS), + wcag_evidence=_wcag(item, ()), + raw_source=item, + parse_status="parsed", + ) + ) + return tuple(normalized) diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py new file mode 100644 index 0000000..f60cca6 --- /dev/null +++ b/vision_aid/evaluation/pricing.py @@ -0,0 +1,71 @@ +"""Reproducible cost estimates from a versioned public price schedule.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any + +DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.v1.json") +MILLION = Decimal(1_000_000) + + +@dataclass(frozen=True) +class PriceSchedule: + """Immutable model rates and their content identity.""" + + data: dict[str, Any] + identity: str + + @classmethod + def load(cls, path: Path) -> PriceSchedule: + raw = path.read_bytes() + data = json.loads(raw) + if not isinstance(data, dict) or not isinstance(data.get("models"), dict): + raise TypeError("Price schedule must contain a models object.") + return cls(data=data, identity=hashlib.sha256(raw).hexdigest()) + + @classmethod + def default(cls) -> PriceSchedule: + return cls.load(DEFAULT_SCHEDULE) + + @property + def version(self) -> str: + return str(self.data["version"]) + + def estimate(self, model: str, usage: Mapping[str, int]) -> str: + """Price reported categories without rounding away tie precision.""" + try: + rates = self.data["models"][model] + except KeyError as error: + raise ValueError( + f"No versioned pricing exists for model {model!r}." + ) from error + input_tokens = Decimal(usage.get("input_tokens", 0)) + cached_tokens = Decimal(usage.get("cached_input_tokens", 0)) + cache_creation_tokens = Decimal(usage.get("cache_creation_input_tokens", 0)) + output_tokens = Decimal(usage.get("output_tokens", 0)) + uncached_tokens = max(Decimal(0), input_tokens - cached_tokens) + input_multiplier = Decimal(1) + output_multiplier = Decimal(1) + threshold = Decimal(rates.get("long_context_threshold_tokens", 0)) + if threshold and input_tokens > threshold: + input_multiplier = Decimal(rates["long_context_input_multiplier"]) + output_multiplier = Decimal(rates["long_context_output_multiplier"]) + cost = ( + uncached_tokens * Decimal(rates["input_per_million_usd"]) * input_multiplier + + cached_tokens + * Decimal(rates["cached_input_per_million_usd"]) + * input_multiplier + + cache_creation_tokens + * Decimal(rates["cache_creation_input_per_million_usd"]) + * input_multiplier + + output_tokens + * Decimal(rates["output_per_million_usd"]) + * output_multiplier + ) / MILLION + return format(cost.normalize(), "f") diff --git a/vision_aid/evaluation/pricing.v1.json b/vision_aid/evaluation/pricing.v1.json new file mode 100644 index 0000000..2a33d66 --- /dev/null +++ b/vision_aid/evaluation/pricing.v1.json @@ -0,0 +1,16 @@ +{ + "models": { + "gpt-5.6-luna": { + "cache_creation_input_per_million_usd": "0.25", + "cached_input_per_million_usd": "0.02", + "input_per_million_usd": "0.20", + "long_context_input_multiplier": "2", + "long_context_output_multiplier": "1.5", + "long_context_threshold_tokens": 272000, + "output_per_million_usd": "1.20" + } + }, + "published_at": "2026-07-09", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", + "version": "openai-public-2026-07-09" +} diff --git a/vision_aid/evaluation/programmatic.py b/vision_aid/evaluation/programmatic.py new file mode 100644 index 0000000..a683da2 --- /dev/null +++ b/vision_aid/evaluation/programmatic.py @@ -0,0 +1,40 @@ +"""Adapter around the production programmatic accessibility checks.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from processing_scripts.programmatic.forms_checklist_02 import audit_forms +from processing_scripts.programmatic.nontext_checklist_03 import audit_nontext +from processing_scripts.programmatic.semantic_checklist_01 import audit_html_file +from vision_aid.evaluation.normalization import normalize_programmatic_findings +from vision_aid.evaluation.schemas import CanonicalFinding + + +@dataclass(frozen=True) +class ProgrammaticAudit: + """Raw checker evidence and its canonical comparison records.""" + + raw_findings: tuple[dict[str, Any], ...] + findings: tuple[CanonicalFinding, ...] + + +def run_programmatic_audit( + html_path: Path, + *, + run_id: str, + page_url: str, +) -> ProgrammaticAudit: + """Run the existing three checker entry points against a benchmark file.""" + path = str(html_path) + raw = tuple(audit_html_file(path) + audit_forms(path) + audit_nontext(path)) + return ProgrammaticAudit( + raw_findings=raw, + findings=normalize_programmatic_findings( + raw, + run_id=run_id, + page_url=page_url, + ), + ) diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/references.py new file mode 100644 index 0000000..0de44b0 --- /dev/null +++ b/vision_aid/evaluation/references.py @@ -0,0 +1,366 @@ +"""Import workbook rows into traceable homepage reference defects.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import replace +from pathlib import Path +from typing import Literal, Protocol +from urllib.parse import urlsplit, urlunsplit + +import openpyxl +from openpyxl.worksheet.datavalidation import DataValidation + +from vision_aid.evaluation.schemas import ( + ReferenceDefect, + ReferenceSet, + ReviewProvenance, +) + + +class ReferenceSetError(ValueError): + """The private workbook cannot produce a trustworthy reference set.""" + + +class EligibilityReviewer(Protocol): + """Adapter contract for human or future model eligibility review.""" + + def review(self, reference_set: ReferenceSet) -> ReferenceSet: + """Return the reference set with validated review decisions.""" + + +_ALIASES = { + "scope": ("page", "page scope", "scope", "page name"), + "url": ("url", "page url", "web address"), + "problem": ("problem", "issue", "defect", "description"), + "location": ("location", "element", "where"), + "wcag": ("wcag", "wcag criteria", "success criterion", "criterion"), + "recommendation": ("recommendation", "remediation", "fix"), +} +ELIGIBILITY_VALUES = ( + "llm_eligible", + "programmatic", + "unavailable_evidence", + "ambiguous", +) +DECISION_VALUES = ("accepted", "rejected", "needs_review") +ELIGIBILITY_COLUMNS = ( + "reference_id", + "source_sheet", + "source_row", + "page_scope", + "canonical_url", + "problem", + "location", + "wcag_evidence", + "recommendation", + "raw_evidence", + "classification", + "decision", + "rationale", + "reviewer", + "confidence", + "timestamp", + "required_subdefects", + "notes", +) + + +def _text(value: object) -> str: + if value is None: + return "" + return str(value) + + +def _canonical_url(value: str) -> str: + parts = urlsplit(value.strip()) + if not parts.scheme or not parts.netloc: + return value.strip() + path = parts.path or "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "", "")) + + +def _field(headers: dict[str, str], raw: dict[str, str], name: str) -> str: + for alias in _ALIASES[name]: + header = headers.get(alias) + if header is not None: + return raw.get(header, "") + return "" + + +def _wcag_values(value: str) -> tuple[str, ...]: + return tuple(dict.fromkeys(re.findall(r"\b\d+\.\d+\.\d+\b", value))) + + +def import_homepage_references( + workbook_path: Path, + *, + expected_sha256: str, + homepage_url: str, + version: str, +) -> ReferenceSet: + """Import every Home and Global row, retaining source text verbatim.""" + if not workbook_path.is_file(): + raise ReferenceSetError( + f"Private workbook not found at {workbook_path}. Obtain it from an " + "authorized project source or pass an explicit local path." + ) + actual_sha256 = hashlib.sha256(workbook_path.read_bytes()).hexdigest() + if actual_sha256.lower() != expected_sha256.lower(): + raise ReferenceSetError( + "Private workbook checksum drift detected: expected " + f"{expected_sha256}, found {actual_sha256}. Review and record the new " + "authorized workbook before importing it." + ) + + try: + workbook = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + except (OSError, ValueError, TypeError) as error: + raise ReferenceSetError(f"Could not read private workbook: {error}") from error + + canonical_homepage = _canonical_url(homepage_url) + imported: list[ReferenceDefect] = [] + for sheet in workbook.worksheets: + rows = sheet.iter_rows(values_only=True) + first = next(rows, None) + if first is None: + continue + header_names = [_text(value).strip() for value in first] + normalized_headers = { + header.casefold(): header for header in header_names if header + } + if not any(alias in normalized_headers for alias in _ALIASES["scope"]): + continue + for row_number, values in enumerate(rows, start=2): + raw = { + header: _text(value) + for header, value in zip(header_names, values, strict=False) + if header + } + scope_text = _field(normalized_headers, raw, "scope").strip().casefold() + row_url = _canonical_url(_field(normalized_headers, raw, "url")) + is_global = scope_text == "global" or row_url.casefold() in { + "global", + "all pages", + "sitewide", + "site-wide", + } + is_home = scope_text in {"home", "homepage", "home page"} + if not is_home and not is_global and row_url != canonical_homepage: + continue + page_scope: Literal["home", "global"] = "global" if is_global else "home" + identity = f"{actual_sha256}:{sheet.title}:{row_number}".encode() + imported.append( + ReferenceDefect( + reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], + workbook_filename=workbook_path.name, + workbook_sha256=actual_sha256, + source_sheet=sheet.title, + source_row=row_number, + raw_evidence=raw, + canonical_url=canonical_homepage, + page_scope=page_scope, + problem=_field(normalized_headers, raw, "problem"), + location=_field(normalized_headers, raw, "location"), + wcag_evidence=_wcag_values(_field(normalized_headers, raw, "wcag")), + recommendation=_field(normalized_headers, raw, "recommendation"), + ) + ) + workbook.close() + if not imported: + raise ReferenceSetError( + "The workbook contained no Home or applicable Global rows. Check the " + "homepage URL and workbook column names." + ) + return ReferenceSet( + version=version, + workbook_filename=workbook_path.name, + workbook_sha256=actual_sha256, + homepage_url=canonical_homepage, + references=tuple(imported), + ) + + +def export_eligibility_workbook( + reference_set: ReferenceSet, + output_path: Path, +) -> None: + """Create a private, multiline-friendly eligibility review workbook.""" + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Eligibility" + sheet.append(ELIGIBILITY_COLUMNS) + for reference in reference_set.references: + sheet.append( + ( + reference.reference_id, + reference.source_sheet, + reference.source_row, + reference.page_scope, + reference.canonical_url, + reference.problem, + reference.location, + ", ".join(reference.wcag_evidence), + reference.recommendation, + json.dumps(reference.raw_evidence, ensure_ascii=False, sort_keys=True), + reference.eligibility or "", + reference.eligibility_state or "needs_review", + ( + reference.eligibility_review.rationale + if reference.eligibility_review + else "" + ), + ( + reference.eligibility_review.reviewer + if reference.eligibility_review + else "" + ), + ( + reference.eligibility_review.confidence + if reference.eligibility_review + else "" + ), + ( + reference.eligibility_review.timestamp + if reference.eligibility_review + else "" + ), + "\n".join(reference.required_subdefects), + "", + ) + ) + sheet.freeze_panes = "A2" + sheet.auto_filter.ref = sheet.dimensions + sheet.column_dimensions["F"].width = 45 + sheet.column_dimensions["G"].width = 32 + sheet.column_dimensions["J"].width = 60 + sheet.column_dimensions["M"].width = 45 + for row in sheet.iter_rows(min_row=2): + for cell in row: + cell.alignment = openpyxl.styles.Alignment(wrap_text=True, vertical="top") + classification_validation = DataValidation( + type="list", + formula1='"' + ",".join(ELIGIBILITY_VALUES) + '"', + ) + decision_validation = DataValidation( + type="list", + formula1='"' + ",".join(DECISION_VALUES) + '"', + ) + sheet.add_data_validation(classification_validation) + sheet.add_data_validation(decision_validation) + classification_validation.add(f"K2:K{max(2, sheet.max_row)}") + decision_validation.add(f"L2:L{max(2, sheet.max_row)}") + output_path.parent.mkdir(parents=True, exist_ok=True) + workbook.save(output_path) + + +class HumanEligibilityReviewer: + """Load eligibility decisions from the private review workbook.""" + + def __init__(self, review_path: Path): + self.review_path = review_path + + def review(self, reference_set: ReferenceSet) -> ReferenceSet: + """Validate and apply exactly one decision for every source row.""" + try: + workbook = openpyxl.load_workbook( + self.review_path, + read_only=True, + data_only=True, + ) + sheet = workbook["Eligibility"] + except (OSError, KeyError, ValueError) as error: + raise ReferenceSetError( + f"Could not read Eligibility review workbook: {error}" + ) from error + rows = sheet.iter_rows(values_only=True) + raw_headers = next(rows, None) + headers = [_text(value).strip() for value in raw_headers or ()] + missing_columns = set(ELIGIBILITY_COLUMNS) - set(headers) + if missing_columns: + raise ReferenceSetError( + "Eligibility review is missing columns: " + + ", ".join(sorted(missing_columns)) + ) + decisions: dict[str, dict[str, str]] = {} + for row_number, values in enumerate(rows, start=2): + record = { + header: _text(value).strip() + for header, value in zip(headers, values, strict=False) + } + reference_id = record["reference_id"] + if not reference_id: + continue + if reference_id in decisions: + raise ReferenceSetError( + f"Eligibility review repeats reference {reference_id}." + ) + classification = record["classification"] + state = record["decision"] + if classification not in ELIGIBILITY_VALUES: + raise ReferenceSetError( + f"Eligibility row {row_number} has invalid classification " + f"{classification!r}." + ) + if state not in DECISION_VALUES: + raise ReferenceSetError( + f"Eligibility row {row_number} has invalid decision {state!r}." + ) + for field in ("rationale", "reviewer", "timestamp"): + if not record[field]: + raise ReferenceSetError( + f"Eligibility row {row_number} requires {field}." + ) + try: + confidence = float(record["confidence"]) + except ValueError as error: + raise ReferenceSetError( + f"Eligibility row {row_number} requires numeric confidence." + ) from error + if not 0 <= confidence <= 1: + raise ReferenceSetError( + f"Eligibility row {row_number} confidence must be 0 through 1." + ) + record["confidence"] = str(confidence) + decisions[reference_id] = record + workbook.close() + + expected_ids = {item.reference_id for item in reference_set.references} + if set(decisions) != expected_ids: + missing = sorted(expected_ids - set(decisions)) + unknown = sorted(set(decisions) - expected_ids) + details = [] + if missing: + details.append("missing " + ", ".join(missing)) + if unknown: + details.append("unknown " + ", ".join(unknown)) + raise ReferenceSetError( + "Eligibility review must decide every reference exactly once: " + + "; ".join(details) + ) + + reviewed = [] + for reference in reference_set.references: + decision = decisions[reference.reference_id] + subdefects = tuple( + value.strip() + for value in decision["required_subdefects"].splitlines() + if value.strip() + ) + reviewed.append( + replace( + reference, + eligibility=decision["classification"], # type: ignore[arg-type] + eligibility_state=decision["decision"], # type: ignore[arg-type] + eligibility_review=ReviewProvenance( + reviewer=decision["reviewer"], + rationale=decision["rationale"], + confidence=float(decision["confidence"]), + timestamp=decision["timestamp"], + ), + required_subdefects=subdefects, + ) + ) + return replace(reference_set, references=tuple(reviewed)) diff --git a/vision_aid/evaluation/reporting.py b/vision_aid/evaluation/reporting.py new file mode 100644 index 0000000..a9682b2 --- /dev/null +++ b/vision_aid/evaluation/reporting.py @@ -0,0 +1,90 @@ +"""JSON, CSV, and Markdown projections of one evaluation score.""" + +from __future__ import annotations + +import csv +import json +from dataclasses import asdict, dataclass +from decimal import Decimal +from pathlib import Path + +from vision_aid.evaluation.scoring import EvaluationScore + + +@dataclass(frozen=True) +class ReportPaths: + """Files written for one score object.""" + + json: Path + csv: Path + markdown: Path + + +def _percentage(metric: dict[str, int | str]) -> str: + return f"{Decimal(str(metric['rate'])) * 100:.2f}%" + + +def write_reports(score: EvaluationScore, output_directory: Path) -> ReportPaths: + """Write three consistent views without recalculating any score.""" + output_directory.mkdir(parents=True, exist_ok=True) + base = output_directory / score.run_id + json_path = base.with_suffix(".json") + csv_path = base.with_suffix(".csv") + markdown_path = base.with_suffix(".md") + json_path.write_text( + json.dumps(score.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + row_dicts = [asdict(row) for row in score.rows] + with csv_path.open("w", newline="", encoding="utf-8") as stream: + if row_dicts: + writer = csv.DictWriter(stream, fieldnames=list(row_dicts[0])) + writer.writeheader() + for data_row in row_dicts: + data_row["model_finding_ids"] = "; ".join(data_row["model_finding_ids"]) + data_row["programmatic_finding_ids"] = "; ".join( + data_row["programmatic_finding_ids"] + ) + writer.writerow(data_row) + recall = score.workbook_row_recall + programmatic = score.programmatic_coverage + combined = score.combined_workbook_coverage + lines = [ + f"# Illustrative model evaluation: {score.model}", + "", + ( + "> This homepage result is illustrative and does not establish broad " + "small-versus-large model equivalence." + ), + "", + ( + f"- Workbook-row recall: {recall['caught']}/{recall['denominator']} " + f"({_percentage(recall)})" + ), + ( + f"- Programmatic coverage: {programmatic['caught']}/" + f"{programmatic['denominator']} ({_percentage(programmatic)})" + ), + ( + f"- Combined workbook coverage: {combined['caught']}/" + f"{combined['denominator']} ({_percentage(combined)})" + ), + f"- Estimated audit cost: ${score.audit_cost_usd}", + f"- Estimated evaluation cost: ${score.evaluation_cost_usd}", + f"- Estimated total experiment cost: ${score.total_experiment_cost_usd}", + f"- Complete and rankable: {score.rankable}", + f"- Format failures: {len(score.parse_failures)}", + f"- Unmatched audit findings: {len(score.unmatched_audit_finding_ids)}", + "", + "| Source row | Eligibility | Programmatic | Model | Disposition |", + "|---|---|---:|---:|---|", + ] + for report_row in score.rows: + lines.append( + f"| {report_row.source_sheet}!{report_row.source_row} | " + f"{report_row.eligibility} | {report_row.programmatic_caught} | " + f"{report_row.model_caught} | {report_row.disposition} |" + ) + markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return ReportPaths(json=json_path, csv=csv_path, markdown=markdown_path) diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py new file mode 100644 index 0000000..b84eae1 --- /dev/null +++ b/vision_aid/evaluation/schemas.py @@ -0,0 +1,122 @@ +"""Canonical, JSON-compatible records shared by evaluation stages.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Literal + +Eligibility = Literal[ + "llm_eligible", + "programmatic", + "unavailable_evidence", + "ambiguous", +] +DecisionState = Literal["accepted", "rejected", "needs_review"] + + +@dataclass(frozen=True) +class ReviewProvenance: + """Who made a review decision and why.""" + + reviewer: str + rationale: str + confidence: float + timestamp: str + + +@dataclass(frozen=True) +class ReferenceDefect: + """One workbook source row retained as one benchmark scoring unit.""" + + reference_id: str + workbook_filename: str + workbook_sha256: str + source_sheet: str + source_row: int + raw_evidence: dict[str, str] + canonical_url: str + page_scope: Literal["home", "global"] + problem: str + location: str + wcag_evidence: tuple[str, ...] + recommendation: str = "" + required_subdefects: tuple[str, ...] = () + eligibility: Eligibility | None = None + eligibility_state: DecisionState | None = None + eligibility_review: ReviewProvenance | None = None + + +@dataclass(frozen=True) +class ReferenceSet: + """Versioned collection of imported or reviewed reference defects.""" + + version: str + workbook_filename: str + workbook_sha256: str + homepage_url: str + references: tuple[ReferenceDefect, ...] + + def to_dict(self) -> dict[str, Any]: + """Return a stable JSON-compatible representation.""" + return asdict(self) + + +@dataclass(frozen=True) +class CanonicalFinding: + """Lossless normalized output from one audit or programmatic source.""" + + finding_id: str + source: Literal["audit", "programmatic"] + run_id: str + model: str | None + prompt: str + checklist: str + page_url: str + problem_family: str + problem: str + element: str + location: str + wcag_evidence: tuple[str, ...] + raw_source: Any + parse_status: str + + +@dataclass(frozen=True) +class MatchDecision: + """Reviewed relationship between one finding and one reference defect.""" + + reference_id: str + finding_id: str + state: DecisionState + provenance: ReviewProvenance + required_subdefect: str | None = None + page_compatible: bool | None = None + failure_compatible: bool | None = None + location_compatible: bool | None = None + notes: str = "" + + +@dataclass(frozen=True) +class Usage: + """Token categories reported by providers across all attempts.""" + + input_tokens: int = 0 + cached_input_tokens: int = 0 + output_tokens: int = 0 + reasoning_tokens: int = 0 + cache_creation_input_tokens: int = 0 + + +@dataclass(frozen=True) +class RunMetadata: + """Operational facts used by deterministic scoring and comparison.""" + + run_id: str + model: str + complete: bool + comparable_identity: str + audit_cost_usd: str + evaluation_cost_usd: str = "0" + usage: Usage = field(default_factory=Usage) + wall_time_seconds: float = 0 + request_durations_seconds: tuple[float, ...] = () diff --git a/vision_aid/evaluation/scoring.py b/vision_aid/evaluation/scoring.py new file mode 100644 index 0000000..1d35c86 --- /dev/null +++ b/vision_aid/evaluation/scoring.py @@ -0,0 +1,244 @@ +"""Network-free deterministic workbook-row scoring and ranking.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from decimal import Decimal +from typing import Any + +from vision_aid.evaluation.matching import validate_match_decisions +from vision_aid.evaluation.schemas import ( + CanonicalFinding, + MatchDecision, + ReferenceDefect, + ReferenceSet, + RunMetadata, +) + + +def _metric(caught: int, denominator: int) -> dict[str, int | str]: + rate = Decimal(caught) / Decimal(denominator) if denominator else Decimal(0) + return {"caught": caught, "denominator": denominator, "rate": str(rate)} + + +@dataclass(frozen=True) +class RowScore: + """Explain the final disposition for one workbook source row.""" + + reference_id: str + source_sheet: str + source_row: int + page_scope: str + eligibility: str + eligibility_rationale: str + problem: str + location: str + programmatic_caught: bool + model_caught: bool + model_finding_ids: tuple[str, ...] + programmatic_finding_ids: tuple[str, ...] + disposition: str + + +@dataclass(frozen=True) +class EvaluationScore: + """The complete deterministic object from which every report is projected.""" + + run_id: str + model: str + complete: bool + rankable: bool + comparable_identity: str + reference_set_version: str + workbook_sha256: str + workbook_row_recall: dict[str, int | str] + programmatic_coverage: dict[str, int | str] + combined_workbook_coverage: dict[str, int | str] + audit_cost_usd: str + evaluation_cost_usd: str + total_experiment_cost_usd: str + usage: dict[str, int] + wall_time_seconds: float + request_durations_seconds: tuple[float, ...] + unmatched_audit_finding_ids: tuple[str, ...] + parse_failures: tuple[str, ...] + rows: tuple[RowScore, ...] + + def to_dict(self) -> dict[str, Any]: + """Return the stable JSON-compatible score representation.""" + return asdict(self) + + +def _caught_references( + references: Sequence[ReferenceDefect], + decisions: Sequence[MatchDecision], +) -> tuple[set[str], dict[str, tuple[str, ...]]]: + accepted = [item for item in decisions if item.state == "accepted"] + by_reference: dict[str, list[MatchDecision]] = {} + for decision in accepted: + by_reference.setdefault(decision.reference_id, []).append(decision) + caught: set[str] = set() + evidence: dict[str, tuple[str, ...]] = {} + for reference in references: + matches = by_reference.get(reference.reference_id, []) + if reference.required_subdefects: + matched_parts = {item.required_subdefect for item in matches} + is_caught = set(reference.required_subdefects) <= matched_parts + else: + is_caught = bool(matches) + if is_caught: + caught.add(reference.reference_id) + evidence[reference.reference_id] = tuple( + sorted(item.finding_id for item in matches) + ) + return caught, evidence + + +def score_evaluation( + reference_set: ReferenceSet, + audit_findings: Sequence[CanonicalFinding], + programmatic_findings: Sequence[CanonicalFinding], + audit_matches: Sequence[MatchDecision], + programmatic_matches: Sequence[MatchDecision], + run: RunMetadata, + *, + parse_failures: Sequence[str] = (), +) -> EvaluationScore: + """Calculate binary workbook-row credit from approved normalized inputs.""" + unapproved = [ + item.reference_id + for item in reference_set.references + if item.eligibility_state != "accepted" or item.eligibility is None + ] + if unapproved: + raise ValueError( + "Scoring requires accepted eligibility decisions for every reference: " + + ", ".join(unapproved) + ) + audit_matches = validate_match_decisions( + reference_set.references, + audit_findings, + audit_matches, + ) + programmatic_matches = validate_match_decisions( + reference_set.references, + programmatic_findings, + programmatic_matches, + ) + model_caught, model_evidence = _caught_references( + reference_set.references, audit_matches + ) + programmatic_caught, programmatic_evidence = _caught_references( + reference_set.references, programmatic_matches + ) + llm_references = { + item.reference_id + for item in reference_set.references + if item.eligibility == "llm_eligible" + } + programmatic_references = { + item.reference_id + for item in reference_set.references + if item.eligibility == "programmatic" + } + combined = model_caught | programmatic_caught + rows = [] + for reference in reference_set.references: + model_hit = reference.reference_id in model_caught + programmatic_hit = reference.reference_id in programmatic_caught + if model_hit and programmatic_hit: + disposition = "caught_by_both" + elif model_hit: + disposition = "caught_by_model" + elif programmatic_hit: + disposition = "caught_programmatically" + elif reference.eligibility == "programmatic": + disposition = "programmatic_gap" + elif reference.eligibility == "unavailable_evidence": + disposition = "unavailable_evidence" + elif reference.eligibility == "ambiguous": + disposition = "ambiguous" + else: + disposition = "missed_by_model" + rows.append( + RowScore( + reference_id=reference.reference_id, + source_sheet=reference.source_sheet, + source_row=reference.source_row, + page_scope=reference.page_scope, + eligibility=reference.eligibility or "", + eligibility_rationale=( + reference.eligibility_review.rationale + if reference.eligibility_review + else "" + ), + problem=reference.problem, + location=reference.location, + programmatic_caught=programmatic_hit, + model_caught=model_hit, + model_finding_ids=model_evidence[reference.reference_id], + programmatic_finding_ids=programmatic_evidence[reference.reference_id], + disposition=disposition, + ) + ) + accepted_audit_ids = { + item.finding_id for item in audit_matches if item.state == "accepted" + } + total_cost = Decimal(run.audit_cost_usd) + Decimal(run.evaluation_cost_usd) + return EvaluationScore( + run_id=run.run_id, + model=run.model, + complete=run.complete, + rankable=run.complete, + comparable_identity=run.comparable_identity, + reference_set_version=reference_set.version, + workbook_sha256=reference_set.workbook_sha256, + workbook_row_recall=_metric( + len(model_caught & llm_references), len(llm_references) + ), + programmatic_coverage=_metric( + len(programmatic_caught & programmatic_references), + len(programmatic_references), + ), + combined_workbook_coverage=_metric( + len(combined), len(reference_set.references) + ), + audit_cost_usd=run.audit_cost_usd, + evaluation_cost_usd=run.evaluation_cost_usd, + total_experiment_cost_usd=str(total_cost), + usage=asdict(run.usage), + wall_time_seconds=run.wall_time_seconds, + request_durations_seconds=run.request_durations_seconds, + unmatched_audit_finding_ids=tuple( + sorted( + item.finding_id + for item in audit_findings + if item.finding_id not in accepted_audit_ids + ) + ), + parse_failures=tuple(sorted(parse_failures)), + rows=tuple(rows), + ) + + +def rank_scores(scores: Sequence[EvaluationScore]) -> tuple[EvaluationScore, ...]: + """Rank comparable complete runs by recall, then unrounded audit cost only.""" + if any(not item.rankable for item in scores): + raise ValueError("An incomplete run is unranked and cannot be compared.") + identities = {item.comparable_identity for item in scores} + if len(identities) > 1: + raise ValueError( + "Runs use incompatible benchmark versions; cross-version analysis must " + "be explicitly requested and labeled." + ) + return tuple( + sorted( + scores, + key=lambda item: ( + -Decimal(str(item.workbook_row_recall["rate"])), + Decimal(item.audit_cost_usd), + item.run_id, + ), + ) + ) diff --git a/vision_aid/evaluation/serialization.py b/vision_aid/evaluation/serialization.py new file mode 100644 index 0000000..135fb92 --- /dev/null +++ b/vision_aid/evaluation/serialization.py @@ -0,0 +1,87 @@ +"""Validated JSON serialization at private evaluation workflow boundaries.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from vision_aid.evaluation.schemas import ( + CanonicalFinding, + MatchDecision, + ReferenceDefect, + ReferenceSet, + ReviewProvenance, + RunMetadata, + Usage, +) + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + +def save_reference_set(path: Path, reference_set: ReferenceSet) -> None: + write_json(path, reference_set.to_dict()) + + +def _provenance(value: dict[str, Any] | None) -> ReviewProvenance | None: + return ReviewProvenance(**value) if value else None + + +def load_reference_set(path: Path) -> ReferenceSet: + data = json.loads(path.read_text(encoding="utf-8")) + references = [] + for value in data["references"]: + value = dict(value) + value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) + value["required_subdefects"] = tuple(value.get("required_subdefects", ())) + value["eligibility_review"] = _provenance(value.get("eligibility_review")) + references.append(ReferenceDefect(**value)) + return ReferenceSet( + version=data["version"], + workbook_filename=data["workbook_filename"], + workbook_sha256=data["workbook_sha256"], + homepage_url=data["homepage_url"], + references=tuple(references), + ) + + +def save_findings(path: Path, findings: tuple[CanonicalFinding, ...]) -> None: + write_json(path, [asdict(item) for item in findings]) + + +def load_findings(path: Path) -> tuple[CanonicalFinding, ...]: + values = json.loads(path.read_text(encoding="utf-8")) + findings = [] + for value in values: + value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) + findings.append(CanonicalFinding(**value)) + return tuple(findings) + + +def save_match_decisions(path: Path, decisions: tuple[MatchDecision, ...]) -> None: + write_json(path, [asdict(item) for item in decisions]) + + +def load_match_decisions(path: Path) -> tuple[MatchDecision, ...]: + values = json.loads(path.read_text(encoding="utf-8")) + decisions = [] + for value in values: + value["provenance"] = ReviewProvenance(**value["provenance"]) + decisions.append(MatchDecision(**value)) + return tuple(decisions) + + +def load_run_metadata(value: dict[str, Any]) -> RunMetadata: + value = dict(value) + value["usage"] = Usage(**value.get("usage", {})) + value["request_durations_seconds"] = tuple( + value.get("request_durations_seconds", ()) + ) + return RunMetadata(**value) From 8bfe6af71ff9bf1ec176e29faac3d3063cb6f9e5 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 12:27:29 -0400 Subject: [PATCH 016/126] fix(evaluation): enforce execution and review invariants --- README.md | 11 +- processing_scripts/llm_client/client.py | 27 +++- pyproject.toml | 2 +- tests/test_evaluation_audit.py | 174 +++++++++++++++++++++- tests/test_evaluation_cli.py | 186 +++++++++++++++++++++++- tests/test_match_review.py | 16 +- tests/test_normalization.py | 35 ++++- tests/test_programmatic_evaluation.py | 6 +- tests/test_reference_set.py | 19 ++- tests/test_scoring_and_reporting.py | 87 ++++++++++- vision_aid/evaluation/audit.py | 157 +++++++++++++------- vision_aid/evaluation/cli.py | 136 ++++++++++++----- vision_aid/evaluation/matching.py | 69 ++++++--- vision_aid/evaluation/normalization.py | 129 ++++++++++++++-- vision_aid/evaluation/pricing.py | 19 ++- vision_aid/evaluation/programmatic.py | 8 +- vision_aid/evaluation/references.py | 62 ++++++-- vision_aid/evaluation/reporting.py | 104 +++++++++++-- vision_aid/evaluation/schemas.py | 7 + vision_aid/evaluation/scoring.py | 78 ++++++++-- vision_aid/evaluation/serialization.py | 107 +++++++++++++- 21 files changed, 1232 insertions(+), 207 deletions(-) diff --git a/README.md b/README.md index 7010492..d4eb7b9 100644 --- a/README.md +++ b/README.md @@ -305,14 +305,17 @@ extractors, filters, programmatic checks, slicers, and unchanged prompts to write an exact Luna dry-run plan: ```bash -uv run visionaid-evaluate audit +uv run visionaid-evaluate audit \ + --max-audit-cost-usd "$PROPOSED_LIMIT" ``` Review the request count, prompt hashes, estimated input usage, model configuration, pricing identity, and authorization digest in the printed -summary. A billable run requires all of the following: `--live`, the exact -saved plan and digest, `OPENAI_API_KEY`, and a positive cost limit. One -authorization covers the summarized run and its policy-compliant retries: +summary. The live authorization digest binds both the exact plan and proposed +cost limit. A billable run requires all of the following: `--live`, the exact +saved plan and live authorization digest, `OPENAI_API_KEY`, and that same +positive cost limit. The authorization is single-use and covers the summarized +run and its policy-compliant retries: ```bash uv run visionaid-evaluate audit \ diff --git a/processing_scripts/llm_client/client.py b/processing_scripts/llm_client/client.py index 3092b55..139f049 100644 --- a/processing_scripts/llm_client/client.py +++ b/processing_scripts/llm_client/client.py @@ -103,7 +103,8 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: usage : dict ``{"input_tokens": int, "output_tokens": int}`` """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) + serialized_payload = json.dumps(payload, separators=(",", ":")) + filled = prompt_text.replace("{payload}", serialized_payload) request_kwargs = { "model": self.model, @@ -175,7 +176,8 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: usage : dict ``{"input_tokens": int, "output_tokens": int}`` """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) + serialized_payload = json.dumps(payload, separators=(",", ":")) + filled = prompt_text.replace("{payload}", serialized_payload) response = self._client.chat.completions.create( model=self.model, @@ -246,7 +248,8 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: usage : dict ``{"input_tokens": int, "output_tokens": int}`` """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) + serialized_payload = json.dumps(payload, separators=(",", ":")) + filled = prompt_text.replace("{payload}", serialized_payload) response = self._client.models.generate_content( model=self.model, @@ -279,7 +282,19 @@ def create_audit_client( ) -> AuditClient | OpenAIAuditClient | GeminiAuditClient: """Factory that returns the right client class based on the model ID.""" if is_openai_model(model): - return OpenAIAuditClient(model=model, temperature=temperature, max_tokens=max_tokens) + return OpenAIAuditClient( + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) if is_gemini_model(model): - return GeminiAuditClient(model=model, temperature=temperature, max_tokens=max_tokens) - return AuditClient(model=model, temperature=temperature, max_tokens=max_tokens) + return GeminiAuditClient( + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) + return AuditClient( + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) diff --git a/pyproject.toml b/pyproject.toml index c303171..2ec595d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dev = [ pythonpath = ["."] [tool.setuptools.packages.find] -include = ["vision_aid*", "processing_scripts*"] +include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py index a5dc4b7..8f59fe9 100644 --- a/tests/test_evaluation_audit.py +++ b/tests/test_evaluation_audit.py @@ -8,6 +8,7 @@ AuditExecutionError, build_audit_plan, execute_authorized_audit, + live_authorization_digest, ) from vision_aid.evaluation.pricing import PriceSchedule @@ -22,6 +23,7 @@ def _fake_pipeline( progress_callback=None, request_config=None, ) -> dict: + """Write two deterministic prompt artifacts through the adapter seam.""" assert api_key is None assert dry_run is True assert include_summaries is False @@ -32,7 +34,9 @@ def _fake_pipeline( prompts = output_dir / "prompts" prompts.mkdir(parents=True) entries = [] - for index, name in enumerate(("heading_structure", "link_clarity"), start=1): + for index, name in enumerate( + ("heading_structure", "link_clarity"), start=1 + ): prompt_text = f"prompt {index}" (prompts / f"{name}.json").write_text( json.dumps( @@ -57,7 +61,9 @@ def _fake_pipeline( "status": "dry_run", } ) - (output_dir / "programmatic_findings.json").write_text("[]", encoding="utf-8") + (output_dir / "programmatic_findings.json").write_text( + "[]", encoding="utf-8" + ) return { "html_file": html_path, "model": model, @@ -73,8 +79,11 @@ def _fake_pipeline( def test_dry_plan_freezes_configuration_content_and_code_identities( tmp_path: Path, ) -> None: + """A dry plan records the complete frozen benchmark configuration.""" snapshot = tmp_path / "source.html" - snapshot.write_text("Synthetic", encoding="utf-8") + snapshot.write_text( + "Synthetic", encoding="utf-8" + ) snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() run_directory = tmp_path / "run" @@ -103,19 +112,24 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( assert list(plan["prompt_hashes"]) == ["heading_structure", "link_clarity"] assert len(plan["parser_hash"]) == 64 assert len(plan["pricing_identity"]) == 64 - assert len(plan["authorization_digest"]) == 64 + assert len(plan["plan_digest"]) == 64 assert (run_directory / "evaluation-manifest.json").is_file() class FakeClient: def __init__(self, outcomes: list[dict]): + """Queue provider outcomes in request order.""" self.outcomes = outcomes def call(self, prompt: str) -> dict: + """Return the next provider outcome without network access.""" return self.outcomes.pop(0) -def _success(response: str, input_tokens: int = 10, output_tokens: int = 5) -> dict: +def _success( + response: str, input_tokens: int = 10, output_tokens: int = 5 +) -> dict: + """Return one successful synthetic provider result.""" return { "success": True, "response": response, @@ -137,6 +151,7 @@ def _success(response: str, input_tokens: int = 10, output_tokens: int = 5) -> d def _transient() -> dict: + """Return one billed, retryable rate-limit result.""" return { "success": False, "response": None, @@ -158,9 +173,22 @@ def _transient() -> dict: } -def test_live_execution_requires_every_gate_and_records_retries_and_format_failures( +def _permanent(status_code: int, failure_type: str) -> dict: + """Return an unbilled provider rejection that must not be retried.""" + result = _transient() + result["usage"]["input_tokens"] = 0 + result["usage"]["total_tokens"] = 0 + result["failure"] = { + "type": failure_type, + "status_code": status_code, + } + return result + + +def test_live_gates_retries_and_format_failures( tmp_path: Path, ) -> None: + """A live run enforces gates and preserves retries and malformed output.""" snapshot = tmp_path / "source.html" snapshot.write_text("", encoding="utf-8") snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() @@ -176,12 +204,13 @@ def test_live_execution_requires_every_gate_and_records_retries_and_format_failu repository=tmp_path, ) plan_path = run_directory / "evaluation-manifest.json" + authorization = live_authorization_digest(plan["plan_digest"], "1") with pytest.raises(AuditExecutionError, match="--live"): execute_authorized_audit( plan_path, live=False, - authorization=plan["authorization_digest"], + authorization=authorization, api_key="secret", max_audit_cost_usd="1", client=FakeClient([]), @@ -208,7 +237,7 @@ def test_live_execution_requires_every_gate_and_records_retries_and_format_failu manifest = execute_authorized_audit( plan_path, live=True, - authorization=plan["authorization_digest"], + authorization=authorization, api_key="secret", max_audit_cost_usd="1", client=client, @@ -222,9 +251,138 @@ def test_live_execution_requires_every_gate_and_records_retries_and_format_failu assert manifest["usage"]["input_tokens"] == 22 assert manifest["estimated_audit_cost_usd"] == "0.0000164" assert manifest["configuration"]["temperature"] is None + raw_attempts = json.loads( + (run_directory / "raw-attempts.json").read_text(encoding="utf-8") + ) + assert len(raw_attempts["requests"][0]["attempts"]) == 2 + + with pytest.raises(AuditExecutionError, match="already been used"): + execute_authorized_audit( + plan_path, + live=True, + authorization=authorization, + api_key="secret", + max_audit_cost_usd="1", + client=FakeClient([]), + sleep=lambda _delay: None, + ) + + +def test_cost_guard_is_checked_before_each_retry(tmp_path: Path) -> None: + """A billed failure can exhaust the limit before an automatic retry.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() + run_directory = tmp_path / "run" + plan = build_audit_plan( + snapshot, + run_directory, + workbook_sha256="a" * 64, + snapshot_sha256=snapshot_sha256, + reference_set_version="references-v1", + eligibility_identity="c" * 64, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + limit = "0.0000001" + client = FakeClient([_transient(), _success("[]")]) + + manifest = execute_authorized_audit( + run_directory / "evaluation-manifest.json", + live=True, + authorization=live_authorization_digest(plan["plan_digest"], limit), + api_key="secret", + max_audit_cost_usd=limit, + client=client, + sleep=lambda _delay: None, + ) + + assert manifest["complete"] is False + assert manifest["incomplete_reason"] == "audit_cost_limit_exhausted" + assert len(manifest["requests"]) == 1 + assert len(manifest["requests"][0]["attempts"]) == 1 + assert len(client.outcomes) == 1 + + +@pytest.mark.parametrize( + "outcome,expected", + [ + (_permanent(401, "AuthenticationError"), "authentication"), + (_permanent(400, "BadRequestError"), "invalid_request"), + ], +) +def test_permanent_provider_failures_are_not_retried( + tmp_path: Path, + outcome: dict, + expected: str, +) -> None: + """Authentication and invalid requests stop after one recorded attempt.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "run" + plan = build_audit_plan( + snapshot, + run_directory, + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity="c" * 64, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + client = FakeClient([outcome, _success("[]")]) + + manifest = execute_authorized_audit( + run_directory / "evaluation-manifest.json", + live=True, + authorization=live_authorization_digest(plan["plan_digest"], "1"), + api_key="secret", + max_audit_cost_usd="1", + client=client, + sleep=lambda _delay: None, + ) + + assert manifest["incomplete_reason"] == expected + assert len(manifest["requests"][0]["attempts"]) == 1 + assert len(client.outcomes) == 1 + + +def test_exhausted_transient_failures_make_the_run_unranked( + tmp_path: Path, +) -> None: + """Record three transient attempts before marking the run incomplete.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "run" + plan = build_audit_plan( + snapshot, + run_directory, + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity="c" * 64, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + + manifest = execute_authorized_audit( + run_directory / "evaluation-manifest.json", + live=True, + authorization=live_authorization_digest(plan["plan_digest"], "1"), + api_key="secret", + max_audit_cost_usd="1", + client=FakeClient([_transient(), _transient(), _transient()]), + sleep=lambda _delay: None, + ) + + assert manifest["complete"] is False + assert manifest["rankable"] is False + assert manifest["incomplete_reason"] == "transient_failures_exhausted" + assert len(manifest["requests"][0]["attempts"]) == 3 def test_versioned_price_schedule_uses_reported_token_categories() -> None: + """Pricing distinguishes uncached, cached, written, and output tokens.""" schedule = PriceSchedule.default() assert ( schedule.estimate( diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 6207136..8d4f282 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,4 +1,5 @@ import json +import hashlib import os import subprocess import sys @@ -11,16 +12,17 @@ from typing import Callable, Iterator import pytest +import openpyxl PROJECT_ROOT = Path(__file__).parents[1] SNAPSHOT_BODY = ( b'\r\nPristine' b"caf\xc3\xa9\r\n" ) -SNAPSHOT_SHA256 = "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" -TEMPORAL_ASSUMPTION = ( - "Stakeholders report that the homepage has not changed since the workbook audit." +SNAPSHOT_SHA256 = ( + "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" ) +TEMPORAL_ASSUMPTION = "Stakeholders report that the homepage has not changed since the workbook audit." class SnapshotHTTPServer(ThreadingHTTPServer): @@ -366,7 +368,10 @@ def publish_competing_snapshot() -> None: encoding="utf-8", ) - with local_snapshot_server(publish_competing_snapshot) as (source_url, _server): + with local_snapshot_server(publish_competing_snapshot) as ( + source_url, + _server, + ): result = run_cli( "prepare", "--workbook", @@ -380,7 +385,9 @@ def publish_competing_snapshot() -> None: assert result.returncode == 2 assert "appeared during capture" in result.stderr - assert [path.name for path in snapshot_directory.iterdir()] == ["race-marker"] + assert [path.name for path in snapshot_directory.iterdir()] == [ + "race-marker" + ] def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( @@ -400,10 +407,15 @@ def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( ) assert result.returncode == 2 - assert "DAT Vision Aid fixture is not a Pristine benchmark input" in result.stderr + assert ( + "DAT Vision Aid fixture is not a Pristine benchmark input" + in result.stderr + ) -@pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) +@pytest.mark.parametrize( + "api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"] +) def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( tmp_path: Path, synthetic_workbook: Path, @@ -440,3 +452,163 @@ def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( assert result.returncode == 0, result.stderr assert "DRY RUN" in result.stdout assert "No network requests were made" in result.stdout + + +def test_cli_completes_the_synthetic_private_workflow(tmp_path: Path) -> None: + """Prepare, review, audit, and report compose without provider access.""" + workbook_path = tmp_path / "synthetic.xlsx" + workbook = openpyxl.Workbook() + source = workbook.active + source.title = "Defects" + source.append(["Page", "URL", "Problem", "Location", "WCAG"]) + source.append( + [ + "Home", + "https://example.test/", + "Hero alternative text is misleading", + "Hero image", + "1.1.1", + ] + ) + workbook.save(workbook_path) + workbook_sha256 = hashlib.sha256(workbook_path.read_bytes()).hexdigest() + + with local_snapshot_server() as (source_url, _server): + prepared = run_cli( + "prepare", + "--workbook", + str(workbook_path), + "--workbook-sha256", + workbook_sha256, + "--homepage-url", + "https://example.test/", + "--snapshot-url", + source_url, + "--temporal-assumption", + TEMPORAL_ASSUMPTION, + cwd=tmp_path, + ) + assert prepared.returncode == 0, prepared.stderr + + workspace = tmp_path / ".model-evaluation" + eligibility_path = workspace / "reviews" / "eligibility.xlsx" + review = openpyxl.load_workbook(eligibility_path) + eligibility = review["Eligibility"] + columns = {cell.value: cell.column for cell in eligibility[1]} + for name, value in { + "classification": "llm_eligible", + "decision": "accepted", + "rationale": "The model receives the image context.", + "reviewer": "reviewer@example.test", + "confidence": 0.9, + "timestamp": "2026-09-19T12:00:00Z", + }.items(): + eligibility.cell(2, columns[name], value) + review.save(eligibility_path) + + reviewed = run_cli( + "review", + "--eligibility-workbook", + str(eligibility_path), + cwd=tmp_path, + ) + assert reviewed.returncode == 0, reviewed.stderr + + planned = run_cli( + "audit", + "--max-audit-cost-usd", + "0.01", + cwd=tmp_path, + ) + assert planned.returncode == 0, planned.stderr + assert "DRY RUN" in planned.stdout + assert "Live authorization digest" in planned.stdout + + approved_path = workspace / "references" / "approved.json" + approved = json.loads(approved_path.read_text(encoding="utf-8")) + reference_id = approved["references"][0]["reference_id"] + run_directory = workspace / "runs" / "dry-run" + finding_path = run_directory / "synthetic-audit-findings.json" + finding_path.write_text( + json.dumps( + [ + { + "finding_id": "finding-1", + "source": "audit", + "run_id": "synthetic-run", + "model": "gpt-5.6-luna", + "prompt": "informative_alt_quality", + "checklist": "CL03", + "page_url": "https://example.test/", + "problem_family": "informative_alt_quality", + "problem": "Hero alternative text is misleading", + "element": "img", + "location": "Hero image", + "wcag_evidence": ["1.1.1"], + "raw_source": {"issues": ["Misleading alt"]}, + "parse_status": "parsed", + } + ] + ), + encoding="utf-8", + ) + empty_findings = run_directory / "empty-findings.json" + empty_findings.write_text("[]", encoding="utf-8") + audit_matches = run_directory / "audit-matches.json" + audit_matches.write_text( + json.dumps( + [ + { + "reference_id": reference_id, + "finding_id": "finding-1", + "state": "accepted", + "provenance": { + "reviewer": "reviewer@example.test", + "rationale": "Same failure and hero image.", + "confidence": 0.9, + "timestamp": "2026-09-19T12:30:00Z", + }, + "required_subdefect": None, + "page_compatible": True, + "failure_compatible": True, + "location_compatible": True, + "notes": "", + } + ] + ), + encoding="utf-8", + ) + empty_matches = run_directory / "empty-matches.json" + empty_matches.write_text("[]", encoding="utf-8") + plan_path = run_directory / "evaluation-manifest.json" + run_manifest = json.loads(plan_path.read_text(encoding="utf-8")) + run_manifest.update( + { + "complete": True, + "estimated_audit_cost_usd": "0.001", + "usage": {"input_tokens": 10, "output_tokens": 5}, + "wall_time_seconds": 1.5, + "request_durations_seconds": [1.0], + } + ) + synthetic_manifest = run_directory / "synthetic-live-manifest.json" + synthetic_manifest.write_text(json.dumps(run_manifest), encoding="utf-8") + bundle_path = workspace / "synthetic-bundle.json" + bundle_path.write_text( + json.dumps( + { + "reference_set": "references/approved.json", + "audit_findings": "runs/dry-run/synthetic-audit-findings.json", + "programmatic_findings": "runs/dry-run/empty-findings.json", + "audit_matches": "runs/dry-run/audit-matches.json", + "programmatic_matches": "runs/dry-run/empty-matches.json", + "run_manifest": "runs/dry-run/synthetic-live-manifest.json", + } + ), + encoding="utf-8", + ) + + reported = run_cli("report", "--bundle", str(bundle_path), cwd=tmp_path) + + assert reported.returncode == 0, reported.stderr + assert (workspace / "reports" / f"{run_manifest['run_id']}.json").is_file() diff --git a/tests/test_match_review.py b/tests/test_match_review.py index 32ff025..a96256c 100644 --- a/tests/test_match_review.py +++ b/tests/test_match_review.py @@ -11,6 +11,7 @@ def _reference(identifier: str, scope: str = "home") -> ReferenceDefect: + """Build one synthetic image reference defect.""" return ReferenceDefect( reference_id=identifier, workbook_filename="synthetic.xlsx", @@ -26,7 +27,10 @@ def _reference(identifier: str, scope: str = "home") -> ReferenceDefect: ) -def _finding(identifier: str, page: str = "https://example.test/") -> CanonicalFinding: +def _finding( + identifier: str, page: str = "https://example.test/" +) -> CanonicalFinding: + """Build one synthetic canonical audit finding.""" return CanonicalFinding( finding_id=identifier, source="audit", @@ -46,12 +50,15 @@ def _finding(identifier: str, page: str = "https://example.test/") -> CanonicalF def test_candidate_generation_only_removes_hard_incompatibilities() -> None: + """Global evidence remains eligible while different Home pages do not.""" home = _reference("home") global_reference = _reference("global", "global") same_page = _finding("same") other_page = _finding("other", "https://example.test/contact") - candidates = generate_candidates((home, global_reference), (same_page, other_page)) + candidates = generate_candidates( + (home, global_reference), (same_page, other_page) + ) assert ("home", "same") in candidates assert ("home", "other") not in candidates @@ -59,7 +66,10 @@ def test_candidate_generation_only_removes_hard_incompatibilities() -> None: assert ("global", "other") in candidates -def test_human_match_review_round_trips_side_by_side_evidence(tmp_path: Path) -> None: +def test_human_match_review_round_trips_side_by_side_evidence( + tmp_path: Path, +) -> None: + """The Matches workbook returns provenance-complete decisions.""" references = (_reference("ref1"),) findings = (_finding("finding1"),) review_path = tmp_path / "matches.xlsx" diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 4cc6382..0d4e0dd 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -11,12 +11,17 @@ @pytest.mark.parametrize( "prompt_name,output_type", - [(spec.name, spec.output_type) for spec in PROMPT_REGISTRY if not spec.is_summary], + [ + (spec.name, spec.output_type) + for spec in PROMPT_REGISTRY + if not spec.is_summary + ], ) def test_every_active_prompt_has_a_lossless_normalizer( prompt_name: str, output_type: str, ) -> None: + """Every active prompt can emit a canonical evidence-preserving finding.""" item = { "problem": "Synthetic accessibility failure", "location": "hero image", @@ -44,7 +49,8 @@ def test_every_active_prompt_has_a_lossless_normalizer( assert finding.wcag_evidence == ("1.1.1",) -def test_normalization_records_fenced_empty_malformed_and_unknown_responses() -> None: +def test_normalization_records_nonstandard_responses() -> None: + """All structured-output outcomes remain explicit and deterministic.""" fenced = normalize_prompt_response( "link_clarity", '```json\n[{"issue": "Unclear link", "text": "Read more"}]\n```', @@ -83,6 +89,7 @@ def test_normalization_records_fenced_empty_malformed_and_unknown_responses() -> def test_programmatic_normalization_retains_rule_and_raw_source() -> None: + """Programmatic records keep their checker identity and raw evidence.""" raw = { "rule_id": "HEAD_002", "message": "Multiple H1 elements", @@ -100,3 +107,27 @@ def test_programmatic_normalization_retains_rule_and_raw_source() -> None: assert findings[0].source == "programmatic" assert findings[0].problem_family == "HEAD_002" assert findings[0].raw_source == raw + + +def test_normalization_emits_only_failures_from_assessment_responses() -> None: + """Compliant assessments stay observable without becoming findings.""" + clear_link = normalize_prompt_response( + "link_clarity", + '[{"text":"Donate","is_clear":true,"issues":[]}]', + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + table_failure = normalize_prompt_response( + "table_semantics", + '{"is_data_table":true,"caption_clear":false,' + '"issues":["Vague caption"]}', + run_id="run-1", + model="gpt-5.6-luna", + page_url="https://example.test/", + ) + + assert clear_link.parse_status == "empty" + assert clear_link.findings == () + assert table_failure.parse_status == "parsed" + assert table_failure.findings[0].problem == "Vague caption" diff --git a/tests/test_programmatic_evaluation.py b/tests/test_programmatic_evaluation.py index e05a4c3..9953043 100644 --- a/tests/test_programmatic_evaluation.py +++ b/tests/test_programmatic_evaluation.py @@ -6,6 +6,7 @@ def test_existing_programmatic_checks_are_adapted_without_duplicating_rules( tmp_path: Path, ) -> None: + """The evaluation adapter calls all existing deterministic checkers.""" html = tmp_path / "benchmark.html" html.write_text( "Synthetic" @@ -23,4 +24,7 @@ def test_existing_programmatic_checks_are_adapted_without_duplicating_rules( assert "HEAD_002" in rule_ids assert "NON_TEXT_001" in rule_ids assert result.raw_findings - assert all(finding.raw_source in result.raw_findings for finding in result.findings) + assert all( + finding.raw_source in result.raw_findings + for finding in result.findings + ) diff --git a/tests/test_reference_set.py b/tests/test_reference_set.py index af3909c..88fc89a 100644 --- a/tests/test_reference_set.py +++ b/tests/test_reference_set.py @@ -13,15 +13,21 @@ def _write_workbook(path: Path) -> str: + """Write a synthetic multi-scope workbook and return its checksum.""" workbook = openpyxl.Workbook() sheet = workbook.active sheet.title = "Defects" - sheet.append(["Page", "URL", "Problem", "Location", "WCAG", "Recommendation"]) + sheet.append( + ["Page", "URL", "Problem", "Location", "WCAG", "Recommendation"] + ) sheet.append( [ "Home", "https://example.test/", - "Decorative image has descriptive alt text\n(second line retained)", + ( + "Decorative image has descriptive alt text\n" + "(second line retained)" + ), "Hero image", "1.1.1", "Use empty alternative text", @@ -54,6 +60,7 @@ def _write_workbook(path: Path) -> str: def test_import_homepage_references_retains_every_home_and_global_source_row( tmp_path: Path, ) -> None: + """Home and Global imports remain lossless and source-addressed.""" workbook_path = tmp_path / "synthetic.xlsx" checksum = _write_workbook(workbook_path) @@ -65,7 +72,8 @@ def test_import_homepage_references_retains_every_home_and_global_source_row( ) assert [ - (item.source_sheet, item.source_row) for item in reference_set.references + (item.source_sheet, item.source_row) + for item in reference_set.references ] == [ ("Defects", 2), ("Defects", 3), @@ -89,6 +97,7 @@ def test_import_homepage_references_retains_every_home_and_global_source_row( def test_human_eligibility_review_round_trips_validated_provenance( tmp_path: Path, ) -> None: + """Eligibility decisions validate and retain human provenance.""" workbook_path = tmp_path / "synthetic.xlsx" checksum = _write_workbook(workbook_path) reference_set = import_homepage_references( @@ -107,7 +116,9 @@ def test_human_eligibility_review_round_trips_validated_provenance( ("llm_eligible", "accepted", "Human judgment is required."), ("programmatic", "accepted", "A deterministic rule owns this defect."), ] - for row, (classification, state, rationale) in enumerate(decisions, start=2): + for row, (classification, state, rationale) in enumerate( + decisions, start=2 + ): sheet.cell(row, headings["classification"], classification) sheet.cell(row, headings["decision"], state) sheet.cell(row, headings["rationale"], rationale) diff --git a/tests/test_scoring_and_reporting.py b/tests/test_scoring_and_reporting.py index 2cd9e34..5710282 100644 --- a/tests/test_scoring_and_reporting.py +++ b/tests/test_scoring_and_reporting.py @@ -32,6 +32,7 @@ def _reference( *, required_subdefects: tuple[str, ...] = (), ) -> ReferenceDefect: + """Build one approved synthetic reference defect.""" return ReferenceDefect( reference_id=identifier, workbook_filename="synthetic.xlsx", @@ -52,6 +53,7 @@ def _reference( def _finding(identifier: str, source: str = "audit") -> CanonicalFinding: + """Build one synthetic canonical finding.""" return CanonicalFinding( finding_id=identifier, source=source, # type: ignore[arg-type] @@ -70,7 +72,10 @@ def _finding(identifier: str, source: str = "audit") -> CanonicalFinding: ) -def _match(reference: str, finding: str, subdefect: str | None = None) -> MatchDecision: +def _match( + reference: str, finding: str, subdefect: str | None = None +) -> MatchDecision: + """Build one accepted, evidence-compatible match decision.""" return MatchDecision( reference_id=reference, finding_id=finding, @@ -83,9 +88,8 @@ def _match(reference: str, finding: str, subdefect: str | None = None) -> MatchD ) -def test_deterministic_score_separates_model_programmatic_and_combined_coverage() -> ( - None -): +def test_deterministic_score_separates_coverage() -> None: + """Scoring separates coverage and requires every compound sub-defect.""" references = ReferenceSet( version="v1", workbook_filename="synthetic.xlsx", @@ -99,7 +103,11 @@ def test_deterministic_score_separates_model_programmatic_and_combined_coverage( _reference("ref5", "ambiguous"), ), ) - audit_findings = (_finding("finding1"), _finding("finding2"), _finding("extra")) + audit_findings = ( + _finding("finding1"), + _finding("finding2"), + _finding("extra"), + ) programmatic_findings = (_finding("program1", "programmatic"),) run = RunMetadata( run_id="run-1", @@ -136,8 +144,16 @@ def test_deterministic_score_separates_model_programmatic_and_combined_coverage( parse_failures=("heading_structure",), ) - assert partial.workbook_row_recall == {"caught": 1, "denominator": 2, "rate": "0.5"} - assert complete.workbook_row_recall == {"caught": 2, "denominator": 2, "rate": "1"} + assert partial.workbook_row_recall == { + "caught": 1, + "denominator": 2, + "rate": "0.5", + } + assert complete.workbook_row_recall == { + "caught": 2, + "denominator": 2, + "rate": "1", + } assert complete.programmatic_coverage == { "caught": 1, "denominator": 1, @@ -170,6 +186,7 @@ def test_deterministic_score_separates_model_programmatic_and_combined_coverage( def test_match_conflicts_and_incomplete_runs_cannot_be_ranked() -> None: + """Conflicting matches and incomplete runs fail closed.""" references = ReferenceSet( version="v1", workbook_filename="synthetic.xlsx", @@ -204,6 +221,7 @@ def test_match_conflicts_and_incomplete_runs_cannot_be_ranked() -> None: def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: + """Every format exposes the same totals, latency, and row disposition.""" references = ReferenceSet( version="v1", workbook_filename="synthetic.xlsx", @@ -235,11 +253,19 @@ def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: csv_rows = list(csv.DictReader(stream)) markdown = paths.markdown.read_text(encoding="utf-8") assert json_report["workbook_row_recall"]["caught"] == 1 + assert json_report["request_duration_sum_seconds"] == 0 assert csv_rows[0]["model_caught"] == "True" + assert csv_rows[0]["workbook_recall_caught"] == "1" + assert csv_rows[0]["request_duration_sum_seconds"] == "0" assert "1/1 (100.00%)" in markdown + assert "Input tokens: 0" in markdown + assert "Request duration sum: 0s" in markdown + assert "finding1" in markdown + assert REVIEW.rationale in markdown def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: + """Operational metrics cannot displace cost as the recall tie-breaker.""" references = ReferenceSet( version="v1", workbook_filename="synthetic.xlsx", @@ -254,6 +280,13 @@ def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: complete=True, comparable_identity="benchmark-v1", audit_cost_usd="0.0012349", + workbook_sha256="a" * 64, + snapshot_sha256="snapshot-v1", + reference_set_version="v1", + prompt_hashes_identity="prompts-v1", + parser_identity="parser-v1", + eligibility_identity="eligibility-v1", + configuration_identity="configuration-v1", ) expensive = score_evaluation( references, (finding,), (), (_match("ref1", "finding1"),), (), base_run @@ -277,3 +310,43 @@ def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: "model-cheap", "model-expensive", ] + + incompatible = score_evaluation( + replace(references, version="v2"), + (finding,), + (), + (_match("ref1", "finding1"),), + (), + replace( + base_run, + run_id="incompatible", + model="other", + reference_set_version="v2", + ), + ) + with pytest.raises(ValueError, match="incompatible benchmark"): + rank_scores((expensive, incompatible)) + + for field in ( + "snapshot_sha256", + "prompt_hashes_identity", + "parser_identity", + "eligibility_identity", + "configuration_identity", + ): + changed_run = replace( + base_run, + run_id=f"changed-{field}", + model="other", + **{field: "changed"}, + ) + changed = score_evaluation( + references, + (finding,), + (), + (_match("ref1", "finding1"),), + (), + changed_run, + ) + with pytest.raises(ValueError, match="incompatible benchmark"): + rank_scores((expensive, changed)) diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index be81115..fa84751 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -13,12 +13,16 @@ from pathlib import Path from typing import Any, Protocol -from processing_scripts.llm_client.audit import AuditRequestClient, AuditRequestConfig +from processing_scripts.llm_client.audit import ( + AuditRequestClient, + AuditRequestConfig, +) from vision_aid.evaluation.normalization import ( normalize_programmatic_findings, normalize_prompt_response, ) from vision_aid.evaluation.pricing import PriceSchedule +from vision_aid.evaluation.serialization import json_digest, write_json MODEL = "gpt-5.6-luna" REQUEST_CONFIG = AuditRequestConfig( @@ -49,20 +53,12 @@ def call(self, prompt: str) -> dict[str, Any]: def _sha256(path: Path) -> str: + """Hash one artifact without transforming its bytes.""" return hashlib.sha256(path.read_bytes()).hexdigest() -def _json_digest(value: object) -> str: - encoded = json.dumps( - value, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ).encode() - return hashlib.sha256(encoded).hexdigest() - - def _repository_state(repository: Path) -> dict[str, Any]: + """Return the source-control commit and dirty state when available.""" try: commit = subprocess.run( ["git", "rev-parse", "HEAD"], @@ -136,24 +132,22 @@ def build_audit_plan( price_schedule = schedule or PriceSchedule.default() parser_path = Path(__file__).with_name("normalization.py") now = datetime.now(UTC).isoformat().replace("+00:00", "Z") - run_id = "dry-" + _json_digest([snapshot_sha256, now])[:16] + run_id = "dry-" + json_digest([snapshot_sha256, now])[:16] raw_programmatic_path = output_directory / "programmatic_findings.json" - raw_programmatic = json.loads(raw_programmatic_path.read_text(encoding="utf-8")) + raw_programmatic = json.loads( + raw_programmatic_path.read_text(encoding="utf-8") + ) canonical_programmatic = normalize_programmatic_findings( raw_programmatic, run_id=run_id, page_url=homepage_url, ) - canonical_programmatic_path = output_directory / "canonical-programmatic.json" - canonical_programmatic_path.write_text( - json.dumps( - [asdict(item) for item in canonical_programmatic], - indent=2, - sort_keys=True, - ensure_ascii=False, - ) - + "\n", - encoding="utf-8", + canonical_programmatic_path = ( + output_directory / "canonical-programmatic.json" + ) + write_json( + canonical_programmatic_path, + [asdict(item) for item in canonical_programmatic], ) plan: dict[str, Any] = { "kind": "model-evaluation-audit-plan", @@ -188,11 +182,13 @@ def build_audit_plan( "repository": _repository_state(repository or Path.cwd()), "programmatic_findings": { "count": len(canonical_programmatic), - "path": str(canonical_programmatic_path.relative_to(output_directory)), + "path": str( + canonical_programmatic_path.relative_to(output_directory) + ), "sha256": _sha256(canonical_programmatic_path), }, } - plan["benchmark_identity"] = _json_digest( + plan["benchmark_identity"] = json_digest( { key: plan[key] for key in ( @@ -208,15 +204,21 @@ def build_audit_plan( ) } ) - plan["authorization_digest"] = _json_digest(plan) - (output_directory / "evaluation-manifest.json").write_text( - json.dumps(plan, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + plan["plan_digest"] = json_digest(plan) + write_json(output_directory / "evaluation-manifest.json", plan) return plan +def live_authorization_digest(plan_digest: str, maximum_cost_usd: str) -> str: + """Bind one explicit authorization to a plan and cost guardrail.""" + cost = format(Decimal(maximum_cost_usd).normalize(), "f") + return json_digest( + {"plan_digest": plan_digest, "maximum_audit_cost_usd": cost} + ) + + def _failure_classification(result: dict[str, Any]) -> str: + """Classify a provider failure for the bounded retry policy.""" failure = result.get("failure") or {} status = failure.get("status_code") failure_type = str(failure.get("type") or "").casefold() @@ -230,7 +232,11 @@ def _failure_classification(result: dict[str, Any]) -> str: return "timeout" if isinstance(status, int) and status >= 500: return "server_error" - if status in {401, 403} or "auth" in failure_type or "permission" in failure_type: + if ( + status in {401, 403} + or "auth" in failure_type + or "permission" in failure_type + ): return "authentication" if isinstance(status, int) and 400 <= status < 500: return "invalid_request" @@ -238,6 +244,7 @@ def _failure_classification(result: dict[str, Any]) -> str: def _empty_usage() -> dict[str, int]: + """Create the token-category accumulator used across attempts.""" return { "input_tokens": 0, "cached_input_tokens": 0, @@ -248,6 +255,7 @@ def _empty_usage() -> dict[str, int]: def _add_usage(total: dict[str, int], result: dict[str, Any]) -> None: + """Add API-reported categories from one possibly billed attempt.""" usage = result.get("usage") or {} for key in total: total[key] += int(usage.get(key, 0) or 0) @@ -264,14 +272,17 @@ def execute_authorized_audit( schedule: PriceSchedule | None = None, sleep: Callable[[float], None] = time.sleep, ) -> dict[str, Any]: - """Execute the exact reviewed plan only after every spending gate passes.""" + """Execute an exact reviewed plan after every spending gate passes.""" if not live: - raise AuditExecutionError("Live execution requires the explicit --live flag.") + raise AuditExecutionError( + "Live execution requires the explicit --live flag." + ) if not api_key: raise AuditExecutionError("Live execution requires OPENAI_API_KEY.") if max_audit_cost_usd is None: raise AuditExecutionError( - "Live execution requires --max-audit-cost-usd as a spending guardrail." + "Live execution requires --max-audit-cost-usd as a spending " + "guardrail." ) try: cost_limit = Decimal(max_audit_cost_usd) @@ -280,18 +291,25 @@ def execute_authorized_audit( "The audit cost limit must be a decimal amount." ) from error if cost_limit <= 0: - raise AuditExecutionError("The audit cost limit must be greater than zero.") + raise AuditExecutionError( + "The audit cost limit must be greater than zero." + ) plan = json.loads(plan_path.read_text(encoding="utf-8")) - recorded_digest = plan.get("authorization_digest") + recorded_digest = plan.get("plan_digest") digest_input = dict(plan) - digest_input.pop("authorization_digest", None) - if recorded_digest != _json_digest(digest_input): + digest_input.pop("plan_digest", None) + if recorded_digest != json_digest(digest_input): raise AuditExecutionError( "The saved dry-run summary has changed since creation." ) - if authorization != recorded_digest: + expected_authorization = live_authorization_digest( + str(recorded_digest), + str(cost_limit), + ) + if authorization != expected_authorization: raise AuditExecutionError( - "Live authorization does not match the exact summarized configuration." + "Live authorization does not match the summarized configuration " + "and cost guardrail." ) if plan.get("configuration") != { "model": MODEL, @@ -315,6 +333,17 @@ def execute_authorized_audit( request_config=REQUEST_CONFIG, ) output_directory = plan_path.parent + authorization_receipt = output_directory / ( + f".authorization-{expected_authorization}.used" + ) + try: + authorization_receipt.touch(exist_ok=False) + except FileExistsError as error: + raise AuditExecutionError( + "This live authorization has already been used; a rerun requires " + "a " + "new dry-run summary and authorization." + ) from error requests: list[dict[str, Any]] = [] totals = _empty_usage() format_failures: list[str] = [] @@ -322,6 +351,7 @@ def execute_authorized_audit( complete = True incomplete_reason: str | None = None started = time.monotonic() + raw_attempts_path = output_directory / "raw-attempts.json" for prompt in plan["prompts"]: current_cost = Decimal(price_schedule.estimate(MODEL, totals)) if current_cost >= cost_limit: @@ -336,14 +366,25 @@ def execute_authorized_audit( hashlib.sha256(prompt_text.encode()).hexdigest() != plan["prompt_hashes"][name] ): - raise AuditExecutionError(f"Prompt content drift detected for {name}.") + raise AuditExecutionError( + f"Prompt content drift detected for {name}." + ) attempts = [] final_result: dict[str, Any] | None = None + budget_exhausted = False for attempt_index in range(3): + current_cost = Decimal(price_schedule.estimate(MODEL, totals)) + if current_cost >= cost_limit: + complete = False + incomplete_reason = "audit_cost_limit_exhausted" + budget_exhausted = True + break result = request_client.call(prompt_text) _add_usage(totals, result) classification = ( - "success" if result.get("success") else _failure_classification(result) + "success" + if result.get("success") + else _failure_classification(result) ) attempts.append( { @@ -353,6 +394,17 @@ def execute_authorized_audit( } ) final_result = result + write_json( + raw_attempts_path, + { + "plan_digest": recorded_digest, + "requests": [ + *requests, + {"name": name, "attempts": attempts}, + ], + "usage": totals, + }, + ) if result.get("success"): break if classification not in RETRYABLE_FAILURES: @@ -361,12 +413,18 @@ def execute_authorized_audit( break if attempt_index < 2: sleep(BACKOFF_SECONDS[attempt_index]) + if budget_exhausted: + if attempts: + requests.append({"name": name, "attempts": attempts}) + break assert final_result is not None request_record = {"name": name, "attempts": attempts} requests.append(request_record) if not final_result.get("success"): complete = False - incomplete_reason = incomplete_reason or "transient_failures_exhausted" + incomplete_reason = ( + incomplete_reason or "transient_failures_exhausted" + ) break normalized = normalize_prompt_response( name, @@ -382,6 +440,7 @@ def execute_authorized_audit( **plan, "kind": "model-evaluation-live-run", "authorized_plan_digest": recorded_digest, + "authorization_digest": expected_authorization, "maximum_audit_cost_usd": str(cost_limit), "completed_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), "complete": complete, @@ -399,13 +458,9 @@ def execute_authorized_audit( "format_failures": format_failures, "canonical_findings": canonical_findings, } - (output_directory / "canonical-audit-findings.json").write_text( - json.dumps(canonical_findings, indent=2, sort_keys=True, ensure_ascii=False) - + "\n", - encoding="utf-8", - ) - (output_directory / "live-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", + write_json( + output_directory / "canonical-audit-findings.json", + canonical_findings, ) + write_json(output_directory / "live-manifest.json", manifest) return manifest diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index a47b6ae..24a0251 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -24,7 +24,7 @@ load_findings, load_match_decisions, load_reference_set, - load_run_metadata, + load_verified_run_metadata, save_match_decisions, save_reference_set, ) @@ -63,7 +63,8 @@ def build_parser() -> argparse.ArgumentParser: type=Path, default=Path(DEFAULT_WORKBOOK_FILENAME), help=( - f"authorized local source workbook (default: ./{DEFAULT_WORKBOOK_FILENAME})" + "authorized local source workbook " + f"(default: ./{DEFAULT_WORKBOOK_FILENAME})" ), ) prepare.add_argument( @@ -72,7 +73,10 @@ def build_parser() -> argparse.ArgumentParser: ) prepare.add_argument( "--homepage-url", - help="canonical Pristine homepage URL used to select Home and Global rows", + help=( + "canonical Pristine homepage URL used to select Home and Global " + "rows" + ), ) prepare.add_argument( "--reference-set-version", @@ -96,11 +100,21 @@ def build_parser() -> argparse.ArgumentParser: "audit", help="inspect the prepared audit plan in network-free dry-run mode", ) - audit.add_argument("--live", action="store_true", help="enable billable execution") - audit.add_argument("--plan", type=Path, help="exact dry-run manifest to execute") - audit.add_argument("--authorize", help="authorization digest shown by dry run") - audit.add_argument("--max-audit-cost-usd", help="required live cost guardrail") - audit.add_argument("--reference-set", type=Path, help="approved reference-set JSON") + audit.add_argument( + "--live", action="store_true", help="enable billable execution" + ) + audit.add_argument( + "--plan", type=Path, help="exact dry-run manifest to execute" + ) + audit.add_argument( + "--authorize", help="authorization digest shown by dry run" + ) + audit.add_argument( + "--max-audit-cost-usd", help="required live cost guardrail" + ) + audit.add_argument( + "--reference-set", type=Path, help="approved reference-set JSON" + ) audit.add_argument( "--run-dir", type=Path, help="private output directory for the plan" ) @@ -115,8 +129,12 @@ def build_parser() -> argparse.ArgumentParser: type=Path, help="apply a completed Eligibility workbook", ) - review.add_argument("--reference-set", type=Path, help="reference-set JSON input") - review.add_argument("--findings", type=Path, help="canonical findings JSON") + review.add_argument( + "--reference-set", type=Path, help="reference-set JSON input" + ) + review.add_argument( + "--findings", type=Path, help="canonical findings JSON" + ) review.add_argument( "--matches-workbook", type=Path, @@ -125,7 +143,9 @@ def build_parser() -> argparse.ArgumentParser: review.add_argument( "--import-matches", action="store_true", - help="import decisions from --matches-workbook instead of generating it", + help=( + "import decisions from --matches-workbook instead of generating it" + ), ) review.set_defaults(handler=_review) @@ -138,7 +158,9 @@ def build_parser() -> argparse.ArgumentParser: type=Path, help="private JSON bundle containing score inputs", ) - report.add_argument("--output-dir", type=Path, help="private report directory") + report.add_argument( + "--output-dir", type=Path, help="private report directory" + ) report.set_defaults(handler=_report) return parser @@ -148,11 +170,13 @@ def _prepare(arguments: argparse.Namespace) -> int: workbook = require_private_workbook(arguments.workbook) workspace = PrivateWorkspace.from_current_directory() workspace.initialize() - print(f"Private evaluation workspace initialized at {workspace.root.resolve()}") + resolved_workspace = workspace.root.resolve() + print(f"Private evaluation workspace initialized at {resolved_workspace}") print(f"Private source workbook: {workbook}") if bool(arguments.workbook_sha256) != bool(arguments.homepage_url): raise PrivateInputError( - "Reference import requires both --workbook-sha256 and --homepage-url." + "Reference import requires both --workbook-sha256 and " + "--homepage-url." ) if arguments.workbook_sha256: reference_set = import_homepage_references( @@ -168,7 +192,9 @@ def _prepare(arguments: argparse.Namespace) -> int: print(f"Imported reference set: {imported_path.resolve()}") print(f"Eligibility review workbook: {review_path.resolve()}") if arguments.temporal_assumption and not arguments.snapshot_url: - raise PrivateInputError("--temporal-assumption requires --snapshot-url.") + raise PrivateInputError( + "--temporal-assumption requires --snapshot-url." + ) if arguments.snapshot_url: snapshot = prepare_benchmark_snapshot( workspace, @@ -189,7 +215,9 @@ def _audit(arguments: argparse.Namespace) -> int: from vision_aid.evaluation.audit import execute_authorized_audit if arguments.plan is None: - raise PrivateInputError("--live requires --plan from a reviewed dry run.") + raise PrivateInputError( + "--live requires --plan from a reviewed dry run." + ) manifest = execute_authorized_audit( arguments.plan, live=True, @@ -206,19 +234,33 @@ def _audit(arguments: argparse.Namespace) -> int: snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" snapshot_path = snapshot_directory / "source.html" metadata_path = snapshot_directory / "metadata.json" - if reference_path.is_file() and snapshot_path.is_file() and metadata_path.is_file(): - from vision_aid.evaluation.audit import build_audit_plan + if ( + reference_path.is_file() + and snapshot_path.is_file() + and metadata_path.is_file() + ): + from vision_aid.evaluation.audit import ( + build_audit_plan, + live_authorization_digest, + ) reference_set = load_reference_set(reference_path) if any( - item.eligibility_state != "accepted" for item in reference_set.references + item.eligibility_state != "accepted" + for item in reference_set.references ): raise PrivateInputError( "The reference set has unresolved eligibility decisions." ) - snapshot_metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - eligibility_identity = hashlib.sha256(reference_path.read_bytes()).hexdigest() - run_directory = arguments.run_dir or workspace.root / "runs" / "dry-run" + snapshot_metadata = json.loads( + metadata_path.read_text(encoding="utf-8") + ) + eligibility_identity = hashlib.sha256( + reference_path.read_bytes() + ).hexdigest() + run_directory = ( + arguments.run_dir or workspace.root / "runs" / "dry-run" + ) plan = build_audit_plan( snapshot_path, run_directory, @@ -230,10 +272,20 @@ def _audit(arguments: argparse.Namespace) -> int: ) print(f"DRY RUN: {plan['request_count']} sequential requests planned.") print(f"Estimated input tokens: {plan['estimated_input_tokens']}") - print(f"Authorization digest: {plan['authorization_digest']}") - print(f"Plan: {(run_directory / 'evaluation-manifest.json').resolve()}") + print(f"Plan digest: {plan['plan_digest']}") + if arguments.max_audit_cost_usd: + authorization = live_authorization_digest( + plan["plan_digest"], + arguments.max_audit_cost_usd, + ) + print(f"Live authorization digest: {authorization}") + print( + f"Plan: {(run_directory / 'evaluation-manifest.json').resolve()}" + ) return 0 - print("DRY RUN: No network requests were made and no API usage was incurred.") + print( + "DRY RUN: No network requests were made and no API usage was incurred." + ) print(f"Private evaluation workspace: {workspace.root.resolve()}") return 0 @@ -246,9 +298,9 @@ def _review(arguments: argparse.Namespace) -> int: reference_path = arguments.reference_set or ( workspace.root / "references" / "imported.json" ) - reviewed = HumanEligibilityReviewer(arguments.eligibility_workbook).review( - load_reference_set(reference_path) - ) + reviewed = HumanEligibilityReviewer( + arguments.eligibility_workbook + ).review(load_reference_set(reference_path)) approved_path = workspace.root / "references" / "approved.json" save_reference_set(approved_path, reviewed) print(f"Validated reference set: {approved_path.resolve()}") @@ -269,10 +321,15 @@ def _review(arguments: argparse.Namespace) -> int: save_match_decisions(output, decisions) print(f"Validated match decisions: {output.resolve()}") else: - export_match_workbook(references, findings, arguments.matches_workbook) - print(f"Matches review workbook: {arguments.matches_workbook.resolve()}") + export_match_workbook( + references, findings, arguments.matches_workbook + ) + resolved_review = arguments.matches_workbook.resolve() + print(f"Matches review workbook: {resolved_review}") return 0 - print(f"Private review workspace: {(workspace.root / 'reviews').resolve()}") + print( + f"Private review workspace: {(workspace.root / 'reviews').resolve()}" + ) return 0 @@ -283,13 +340,22 @@ def _report(arguments: argparse.Namespace) -> int: if arguments.bundle: bundle = json.loads(arguments.bundle.read_text(encoding="utf-8")) base = arguments.bundle.parent + reference_set = load_reference_set(base / bundle["reference_set"]) + if "run_manifest" not in bundle: + raise PrivateInputError( + "Report bundles require run_manifest so benchmark identities " + "can be verified." + ) score = score_evaluation( - load_reference_set(base / bundle["reference_set"]), + reference_set, load_findings(base / bundle["audit_findings"]), load_findings(base / bundle["programmatic_findings"]), load_match_decisions(base / bundle["audit_matches"]), load_match_decisions(base / bundle["programmatic_matches"]), - load_run_metadata(bundle["run"]), + load_verified_run_metadata( + base / bundle["run_manifest"], + reference_set, + ), parse_failures=tuple(bundle.get("parse_failures", ())), ) output_directory = arguments.output_dir or workspace.root / "reports" @@ -298,7 +364,9 @@ def _report(arguments: argparse.Namespace) -> int: print(f"CSV report: {paths.csv.resolve()}") print(f"Markdown report: {paths.markdown.resolve()}") return 0 - print(f"Private report workspace: {(workspace.root / 'reports').resolve()}") + print( + f"Private report workspace: {(workspace.root / 'reports').resolve()}" + ) return 0 diff --git a/vision_aid/evaluation/matching.py b/vision_aid/evaluation/matching.py index bc40673..43002cd 100644 --- a/vision_aid/evaluation/matching.py +++ b/vision_aid/evaluation/matching.py @@ -76,10 +76,19 @@ def generate_candidates( if not page_compatible: continue reference_location = reference.location.casefold() - finding_evidence = f"{finding.element} {finding.location}".casefold() + finding_evidence = ( + f"{finding.element} {finding.location}".casefold() + ) if reference_location and finding_evidence: - # Only treat explicit, disjoint element categories as impossible. - categories = ("image", "heading", "link", "form", "table", "media") + # Only explicit, disjoint element categories are impossible. + categories = ( + "image", + "heading", + "link", + "form", + "table", + "media", + ) ref_categories = { item for item in categories if item in reference_location } @@ -106,7 +115,9 @@ def validate_match_decisions( """Require resolved evidence-backed decisions and one-to-one findings.""" reference_by_id = {item.reference_id: item for item in references} finding_ids = {item.finding_id for item in findings} - pair_counts = Counter((item.reference_id, item.finding_id) for item in decisions) + pair_counts = Counter( + (item.reference_id, item.finding_id) for item in decisions + ) repeated = [pair for pair, count in pair_counts.items() if count > 1] if repeated: raise MatchValidationError( @@ -125,8 +136,13 @@ def validate_match_decisions( raise MatchValidationError( "Unresolved needs_review match decisions cannot be scored." ) - if not decision.provenance.reviewer or not decision.provenance.rationale: - raise MatchValidationError("Every match decision requires provenance.") + if ( + not decision.provenance.reviewer + or not decision.provenance.rationale + ): + raise MatchValidationError( + "Every match decision requires provenance." + ) if not 0 <= decision.provenance.confidence <= 1: raise MatchValidationError("Match confidence must be 0 through 1.") if decision.state != "accepted": @@ -139,19 +155,26 @@ def validate_match_decisions( ) ): raise MatchValidationError( - "An accepted match requires compatible page, underlying failure, " + "An accepted match requires compatible page, underlying " + "failure, " "and location or element-group evidence." ) reference = reference_by_id[decision.reference_id] if reference.required_subdefects: - if decision.required_subdefect not in reference.required_subdefects: + if ( + decision.required_subdefect + not in reference.required_subdefects + ): + reference_id = reference.reference_id raise MatchValidationError( - f"Accepted match for {reference.reference_id} must identify one " + f"Accepted match for {reference_id} must identify " + "one " "of its required sub-defects." ) elif decision.required_subdefect: raise MatchValidationError( - f"Reference {reference.reference_id} has no required sub-defects." + f"Reference {reference.reference_id} has no required " + "sub-defects." ) accepted = [item for item in decisions if item.state == "accepted"] finding_counts = Counter(item.finding_id for item in accepted) @@ -166,7 +189,9 @@ def validate_match_decisions( for item in accepted if not reference_by_id[item.reference_id].required_subdefects ) - conflicts = [item for item, count in plain_reference_counts.items() if count > 1] + conflicts = [ + item for item, count in plain_reference_counts.items() if count > 1 + ] if conflicts: raise MatchValidationError( "Accepted matches must be one-to-one; reference " @@ -182,7 +207,8 @@ def validate_match_decisions( ] if subdefect_conflicts: raise MatchValidationError( - "Accepted matches must be one-to-one at the required sub-defect level." + "Accepted matches must be one-to-one at the required " + "sub-defect level." ) return tuple(decisions) @@ -210,13 +236,17 @@ def export_match_workbook( reference.problem, reference.location, ", ".join(reference.wcag_evidence), - json.dumps(reference.raw_evidence, ensure_ascii=False, sort_keys=True), + json.dumps( + reference.raw_evidence, ensure_ascii=False, sort_keys=True + ), finding.prompt, finding.problem, finding.element, finding.location, ", ".join(finding.wcag_evidence), - json.dumps(finding.raw_source, ensure_ascii=False, sort_keys=True), + json.dumps( + finding.raw_source, ensure_ascii=False, sort_keys=True + ), "needs_review", "", "", @@ -253,10 +283,12 @@ def export_match_workbook( def _text(value: Any) -> str: + """Normalize an optional worksheet cell to stripped text.""" return "" if value is None else str(value).strip() def _boolean(value: Any, *, row: int, field: str) -> bool | None: + """Parse an optional human-entered worksheet boolean.""" if value in (None, ""): return None if isinstance(value, bool): @@ -273,6 +305,7 @@ class HumanMatchReviewer: """Load human match decisions through the shared reviewer contract.""" def __init__(self, review_path: Path): + """Select the private Matches workbook to read.""" self.review_path = review_path def review( @@ -280,7 +313,7 @@ def review( references: Sequence[ReferenceDefect], findings: Sequence[CanonicalFinding], ) -> tuple[MatchDecision, ...]: - """Load provenance-complete decisions and validate accepted conflicts.""" + """Load complete decisions and validate accepted conflicts.""" try: workbook = openpyxl.load_workbook( self.review_path, @@ -298,7 +331,8 @@ def review( missing = set(MATCH_COLUMNS) - set(headers) if missing: raise MatchValidationError( - "Matches review is missing columns: " + ", ".join(sorted(missing)) + "Matches review is missing columns: " + + ", ".join(sorted(missing)) ) decisions = [] for row_number, values in enumerate(rows, start=2): @@ -317,7 +351,8 @@ def review( timestamp = _text(record.get("timestamp")) if not reviewer or not rationale or not timestamp: raise MatchValidationError( - f"Match row {row_number} requires reviewer, rationale, and timestamp." + f"Match row {row_number} requires reviewer, rationale, " + "and timestamp." ) try: raw_confidence = record.get("confidence") diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index d9c4a86..81f94fb 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -23,6 +23,14 @@ "actual_result", "summary", ) +_EXPLICIT_PROBLEM_KEYS = ( + "problem", + "issue", + "issue_title", + "description", + "finding", + "actual_result", +) _LOCATION_KEYS = ( "location", "location_hint", @@ -48,11 +56,13 @@ class NormalizationResult: def _strip_fence(value: str) -> str: + """Remove one optional Markdown JSON fence.""" match = re.fullmatch(r"\s*```(?:json)?\s*(.*?)\s*```\s*", value, re.DOTALL) return match.group(1) if match else value.strip() def _first_text(item: dict[str, Any], keys: Iterable[str]) -> str: + """Return the first populated evidence field as stable text.""" for key in keys: value = item.get(key) if value is None or value == "": @@ -64,6 +74,7 @@ def _first_text(item: dict[str, Any], keys: Iterable[str]) -> str: def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: + """Extract ordered WCAG identifiers or use the prompt defaults.""" values: list[str] = [] for key in _WCAG_KEYS: value = item.get(key) @@ -76,19 +87,98 @@ def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: def _items(parsed: Any, spec: PromptSpec) -> list[dict[str, Any]] | None: + """Return source finding objects from either supported JSON shape.""" if isinstance(parsed, list): return [item for item in parsed if isinstance(item, dict)] if not isinstance(parsed, dict): return None - for key in ("findings", "issues", "problems", "violations", "results"): + for key in ("findings", "violations", "results"): nested = parsed.get(key) - if isinstance(nested, list): + if isinstance(nested, list) and all( + isinstance(item, dict) for item in nested + ): return [item for item in nested if isinstance(item, dict)] - if spec.output_type == "object": - if not parsed: - return [] - return [parsed] - return None + return [parsed] if parsed else [] + + +def _has_values(value: Any) -> bool: + """Treat non-empty strings, mappings, and sequences as evidence.""" + return value not in (None, "", [], {}, ()) + + +def _is_failure(prompt: str, item: dict[str, Any]) -> bool: + """Apply the fixed failure predicate for one active prompt category.""" + if any(_has_values(item.get(key)) for key in _EXPLICIT_PROBLEM_KEYS): + return True + if _has_values(item.get("issues")): + return True + if prompt == "page_title": + return ( + item.get("is_descriptive") is False + or item.get("matches_h1") is False + ) + if prompt == "heading_structure": + return item.get("structure_clear") is False or _has_values( + item.get("vague_headings") + ) + if prompt == "link_clarity": + return item.get("is_clear") is False + if prompt == "table_semantics": + return item.get("caption_clear") is False or _has_values( + item.get("header_clarity_issues") + ) + if prompt == "iframe_titles": + return item.get("is_descriptive") is False + if prompt == "landmark_structure": + return item.get("structure_appropriate") is False + if prompt == "label_quality": + return item.get("is_descriptive") is False + if prompt == "placeholder_as_label": + return True + if prompt == "group_labels": + return item.get("legend_is_meaningful") is False + if prompt == "required_field_indicators": + return ( + item.get("requirement_in_label") is False + and item.get("requirement_in_instructions") is False + ) + if prompt == "form_instructions": + return item.get("instructions_are_helpful") is False + if prompt == "informative_alt_quality": + return item.get("quality") == "poor" + if prompt == "decorative_verification": + return item.get("likely_decorative") is False + if prompt == "actionable_image_alt": + return item.get("describes_action_not_appearance") is False + if prompt == "complex_descriptions": + return item.get("alt_is_sufficient") is False or ( + item.get("long_description_needed") is True + and item.get("long_description_adequate") is not True + ) + if prompt == "svg_accessibility": + return ( + item.get("has_accessible_name") is False + or item.get("title_is_meaningful") is False + ) + if prompt == "icon_font_accessibility": + return item.get("pattern") in {"unlabeled_control", "missing_label"} + if prompt == "media_captions": + return ( + item.get("has_captions_track") is False + or item.get("has_controls") is False + ) + return False + + +def _problem(item: dict[str, Any], prompt_name: str) -> str: + """Choose a readable problem statement while retaining the raw object.""" + explicit = _first_text(item, _EXPLICIT_PROBLEM_KEYS) + if explicit: + return explicit + issues = item.get("issues") + if isinstance(issues, list) and issues: + return "; ".join(str(value) for value in issues) + return _first_text(item, ("reason", "summary")) or prompt_name def normalize_prompt_response( @@ -107,7 +197,9 @@ def normalize_prompt_response( parse_status="unknown_prompt", raw_response=raw_response, findings=(), - error=f"No fixed evaluation normalizer exists for {prompt_name!r}.", + error=( + f"No fixed evaluation normalizer exists for {prompt_name!r}." + ), ) try: parsed = json.loads(_strip_fence(raw_response)) @@ -130,16 +222,21 @@ def normalize_prompt_response( ) findings = [] for index, item in enumerate(items): + if not _is_failure(prompt_name, item): + continue canonical_item = json.dumps( item, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) - identity = f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() + identity = ( + f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() + ) findings.append( CanonicalFinding( - finding_id="finding-" + hashlib.sha256(identity).hexdigest()[:20], + finding_id="finding-" + + hashlib.sha256(identity).hexdigest()[:20], source="audit", run_id=run_id, model=model, @@ -147,7 +244,7 @@ def normalize_prompt_response( checklist=spec.checklist, page_url=page_url, problem_family=prompt_name, - problem=_first_text(item, _PROBLEM_KEYS) or prompt_name, + problem=_problem(item, prompt_name), element=_first_text(item, _ELEMENT_KEYS), location=_first_text(item, _LOCATION_KEYS), wcag_evidence=_wcag(item, spec.wcag_criteria), @@ -178,11 +275,14 @@ def normalize_programmatic_findings( sort_keys=True, separators=(",", ":"), ) - identity = f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() + identity = ( + f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() + ) rule = str(item.get("rule_id") or item.get("rule") or "programmatic") normalized.append( CanonicalFinding( - finding_id="programmatic-" + hashlib.sha256(identity).hexdigest()[:20], + finding_id="programmatic-" + + hashlib.sha256(identity).hexdigest()[:20], source="programmatic", run_id=run_id, model=None, @@ -190,7 +290,8 @@ def normalize_programmatic_findings( checklist=str(item.get("checklist") or "programmatic"), page_url=page_url, problem_family=rule, - problem=_first_text(item, _PROBLEM_KEYS + ("message",)) or rule, + problem=_first_text(item, _PROBLEM_KEYS + ("message",)) + or rule, element=_first_text(item, _ELEMENT_KEYS), location=_first_text(item, _LOCATION_KEYS), wcag_evidence=_wcag(item, ()), diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index f60cca6..1a1ebe1 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -23,18 +23,23 @@ class PriceSchedule: @classmethod def load(cls, path: Path) -> PriceSchedule: + """Load and content-address one JSON price schedule.""" raw = path.read_bytes() data = json.loads(raw) - if not isinstance(data, dict) or not isinstance(data.get("models"), dict): + if not isinstance(data, dict) or not isinstance( + data.get("models"), dict + ): raise TypeError("Price schedule must contain a models object.") return cls(data=data, identity=hashlib.sha256(raw).hexdigest()) @classmethod def default(cls) -> PriceSchedule: + """Load the committed public pricing schedule.""" return cls.load(DEFAULT_SCHEDULE) @property def version(self) -> str: + """Return the curator-assigned price-schedule version.""" return str(self.data["version"]) def estimate(self, model: str, usage: Mapping[str, int]) -> str: @@ -47,7 +52,9 @@ def estimate(self, model: str, usage: Mapping[str, int]) -> str: ) from error input_tokens = Decimal(usage.get("input_tokens", 0)) cached_tokens = Decimal(usage.get("cached_input_tokens", 0)) - cache_creation_tokens = Decimal(usage.get("cache_creation_input_tokens", 0)) + cache_creation_tokens = Decimal( + usage.get("cache_creation_input_tokens", 0) + ) output_tokens = Decimal(usage.get("output_tokens", 0)) uncached_tokens = max(Decimal(0), input_tokens - cached_tokens) input_multiplier = Decimal(1) @@ -55,9 +62,13 @@ def estimate(self, model: str, usage: Mapping[str, int]) -> str: threshold = Decimal(rates.get("long_context_threshold_tokens", 0)) if threshold and input_tokens > threshold: input_multiplier = Decimal(rates["long_context_input_multiplier"]) - output_multiplier = Decimal(rates["long_context_output_multiplier"]) + output_multiplier = Decimal( + rates["long_context_output_multiplier"] + ) cost = ( - uncached_tokens * Decimal(rates["input_per_million_usd"]) * input_multiplier + uncached_tokens + * Decimal(rates["input_per_million_usd"]) + * input_multiplier + cached_tokens * Decimal(rates["cached_input_per_million_usd"]) * input_multiplier diff --git a/vision_aid/evaluation/programmatic.py b/vision_aid/evaluation/programmatic.py index a683da2..9d87e73 100644 --- a/vision_aid/evaluation/programmatic.py +++ b/vision_aid/evaluation/programmatic.py @@ -8,7 +8,9 @@ from processing_scripts.programmatic.forms_checklist_02 import audit_forms from processing_scripts.programmatic.nontext_checklist_03 import audit_nontext -from processing_scripts.programmatic.semantic_checklist_01 import audit_html_file +from processing_scripts.programmatic.semantic_checklist_01 import ( + audit_html_file, +) from vision_aid.evaluation.normalization import normalize_programmatic_findings from vision_aid.evaluation.schemas import CanonicalFinding @@ -29,7 +31,9 @@ def run_programmatic_audit( ) -> ProgrammaticAudit: """Run the existing three checker entry points against a benchmark file.""" path = str(html_path) - raw = tuple(audit_html_file(path) + audit_forms(path) + audit_nontext(path)) + raw = tuple( + audit_html_file(path) + audit_forms(path) + audit_nontext(path) + ) return ProgrammaticAudit( raw_findings=raw, findings=normalize_programmatic_findings( diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/references.py index 0de44b0..e6718c8 100644 --- a/vision_aid/evaluation/references.py +++ b/vision_aid/evaluation/references.py @@ -69,20 +69,25 @@ def review(self, reference_set: ReferenceSet) -> ReferenceSet: def _text(value: object) -> str: + """Retain workbook cell content as text without semantic rewriting.""" if value is None: return "" return str(value) def _canonical_url(value: str) -> str: + """Normalize URL identity while discarding query and fragment drift.""" parts = urlsplit(value.strip()) if not parts.scheme or not parts.netloc: return value.strip() path = parts.path or "/" - return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "", "")) + return urlunsplit( + (parts.scheme.lower(), parts.netloc.lower(), path, "", "") + ) def _field(headers: dict[str, str], raw: dict[str, str], name: str) -> str: + """Read one canonical field through the supported workbook aliases.""" for alias in _ALIASES[name]: header = headers.get(alias) if header is not None: @@ -91,6 +96,7 @@ def _field(headers: dict[str, str], raw: dict[str, str], name: str) -> str: def _wcag_values(value: str) -> tuple[str, ...]: + """Extract ordered WCAG success-criterion identifiers.""" return tuple(dict.fromkeys(re.findall(r"\b\d+\.\d+\.\d+\b", value))) @@ -104,21 +110,27 @@ def import_homepage_references( """Import every Home and Global row, retaining source text verbatim.""" if not workbook_path.is_file(): raise ReferenceSetError( - f"Private workbook not found at {workbook_path}. Obtain it from an " + f"Private workbook not found at {workbook_path}. Obtain it from " + "an " "authorized project source or pass an explicit local path." ) actual_sha256 = hashlib.sha256(workbook_path.read_bytes()).hexdigest() if actual_sha256.lower() != expected_sha256.lower(): raise ReferenceSetError( "Private workbook checksum drift detected: expected " - f"{expected_sha256}, found {actual_sha256}. Review and record the new " + f"{expected_sha256}, found {actual_sha256}. Review and record " + "the new " "authorized workbook before importing it." ) try: - workbook = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + workbook = openpyxl.load_workbook( + workbook_path, read_only=True, data_only=True + ) except (OSError, ValueError, TypeError) as error: - raise ReferenceSetError(f"Could not read private workbook: {error}") from error + raise ReferenceSetError( + f"Could not read private workbook: {error}" + ) from error canonical_homepage = _canonical_url(homepage_url) imported: list[ReferenceDefect] = [] @@ -139,7 +151,9 @@ def import_homepage_references( for header, value in zip(header_names, values, strict=False) if header } - scope_text = _field(normalized_headers, raw, "scope").strip().casefold() + scope_text = ( + _field(normalized_headers, raw, "scope").strip().casefold() + ) row_url = _canonical_url(_field(normalized_headers, raw, "url")) is_global = scope_text == "global" or row_url.casefold() in { "global", @@ -150,11 +164,14 @@ def import_homepage_references( is_home = scope_text in {"home", "homepage", "home page"} if not is_home and not is_global and row_url != canonical_homepage: continue - page_scope: Literal["home", "global"] = "global" if is_global else "home" + page_scope: Literal["home", "global"] = ( + "global" if is_global else "home" + ) identity = f"{actual_sha256}:{sheet.title}:{row_number}".encode() imported.append( ReferenceDefect( - reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], + reference_id="ref-" + + hashlib.sha256(identity).hexdigest()[:16], workbook_filename=workbook_path.name, workbook_sha256=actual_sha256, source_sheet=sheet.title, @@ -164,14 +181,19 @@ def import_homepage_references( page_scope=page_scope, problem=_field(normalized_headers, raw, "problem"), location=_field(normalized_headers, raw, "location"), - wcag_evidence=_wcag_values(_field(normalized_headers, raw, "wcag")), - recommendation=_field(normalized_headers, raw, "recommendation"), + wcag_evidence=_wcag_values( + _field(normalized_headers, raw, "wcag") + ), + recommendation=_field( + normalized_headers, raw, "recommendation" + ), ) ) workbook.close() if not imported: raise ReferenceSetError( - "The workbook contained no Home or applicable Global rows. Check the " + "The workbook contained no Home or applicable Global rows. " + "Check the " "homepage URL and workbook column names." ) return ReferenceSet( @@ -204,7 +226,9 @@ def export_eligibility_workbook( reference.location, ", ".join(reference.wcag_evidence), reference.recommendation, - json.dumps(reference.raw_evidence, ensure_ascii=False, sort_keys=True), + json.dumps( + reference.raw_evidence, ensure_ascii=False, sort_keys=True + ), reference.eligibility or "", reference.eligibility_state or "needs_review", ( @@ -239,7 +263,9 @@ def export_eligibility_workbook( sheet.column_dimensions["M"].width = 45 for row in sheet.iter_rows(min_row=2): for cell in row: - cell.alignment = openpyxl.styles.Alignment(wrap_text=True, vertical="top") + cell.alignment = openpyxl.styles.Alignment( + wrap_text=True, vertical="top" + ) classification_validation = DataValidation( type="list", formula1='"' + ",".join(ELIGIBILITY_VALUES) + '"', @@ -260,6 +286,7 @@ class HumanEligibilityReviewer: """Load eligibility decisions from the private review workbook.""" def __init__(self, review_path: Path): + """Select the private Eligibility workbook to read.""" self.review_path = review_path def review(self, reference_set: ReferenceSet) -> ReferenceSet: @@ -306,7 +333,8 @@ def review(self, reference_set: ReferenceSet) -> ReferenceSet: ) if state not in DECISION_VALUES: raise ReferenceSetError( - f"Eligibility row {row_number} has invalid decision {state!r}." + f"Eligibility row {row_number} has invalid decision " + f"{state!r}." ) for field in ("rationale", "reviewer", "timestamp"): if not record[field]: @@ -317,11 +345,13 @@ def review(self, reference_set: ReferenceSet) -> ReferenceSet: confidence = float(record["confidence"]) except ValueError as error: raise ReferenceSetError( - f"Eligibility row {row_number} requires numeric confidence." + f"Eligibility row {row_number} requires numeric " + "confidence." ) from error if not 0 <= confidence <= 1: raise ReferenceSetError( - f"Eligibility row {row_number} confidence must be 0 through 1." + f"Eligibility row {row_number} confidence must be 0 " + "through 1." ) record["confidence"] = str(confidence) decisions[reference_id] = record diff --git a/vision_aid/evaluation/reporting.py b/vision_aid/evaluation/reporting.py index a9682b2..caf935a 100644 --- a/vision_aid/evaluation/reporting.py +++ b/vision_aid/evaluation/reporting.py @@ -21,10 +21,18 @@ class ReportPaths: def _percentage(metric: dict[str, int | str]) -> str: + """Format one metric rate for the human report.""" return f"{Decimal(str(metric['rate'])) * 100:.2f}%" -def write_reports(score: EvaluationScore, output_directory: Path) -> ReportPaths: +def _markdown(value: object) -> str: + """Keep multiline evidence inside one safe Markdown table cell.""" + return str(value).replace("|", "\\|").replace("\n", "
") + + +def write_reports( + score: EvaluationScore, output_directory: Path +) -> ReportPaths: """Write three consistent views without recalculating any score.""" output_directory.mkdir(parents=True, exist_ok=True) base = output_directory / score.run_id @@ -32,20 +40,64 @@ def write_reports(score: EvaluationScore, output_directory: Path) -> ReportPaths csv_path = base.with_suffix(".csv") markdown_path = base.with_suffix(".md") json_path.write_text( - json.dumps(score.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + json.dumps( + score.to_dict(), indent=2, sort_keys=True, ensure_ascii=False + ) + "\n", encoding="utf-8", ) - row_dicts = [asdict(row) for row in score.rows] + common = { + "run_id": score.run_id, + "model": score.model, + "complete": score.complete, + "rankable": score.rankable, + "workbook_recall_caught": score.workbook_row_recall["caught"], + "workbook_recall_denominator": score.workbook_row_recall[ + "denominator" + ], + "workbook_recall_rate": score.workbook_row_recall["rate"], + "programmatic_coverage_caught": score.programmatic_coverage["caught"], + "programmatic_coverage_denominator": score.programmatic_coverage[ + "denominator" + ], + "programmatic_coverage_rate": score.programmatic_coverage["rate"], + "combined_caught": score.combined_workbook_coverage["caught"], + "combined_denominator": score.combined_workbook_coverage[ + "denominator" + ], + "combined_rate": score.combined_workbook_coverage["rate"], + "audit_cost_usd": score.audit_cost_usd, + "evaluation_cost_usd": score.evaluation_cost_usd, + "total_experiment_cost_usd": score.total_experiment_cost_usd, + **score.usage, + "wall_time_seconds": score.wall_time_seconds, + "request_duration_sum_seconds": score.request_duration_sum_seconds, + "request_durations_seconds": json.dumps( + score.request_durations_seconds + ), + "parse_failure_count": len(score.parse_failures), + "unmatched_audit_finding_count": len( + score.unmatched_audit_finding_ids + ), + } + row_dicts = [{**common, **asdict(row)} for row in score.rows] with csv_path.open("w", newline="", encoding="utf-8") as stream: if row_dicts: writer = csv.DictWriter(stream, fieldnames=list(row_dicts[0])) writer.writeheader() for data_row in row_dicts: - data_row["model_finding_ids"] = "; ".join(data_row["model_finding_ids"]) + data_row["model_finding_ids"] = "; ".join( + data_row["model_finding_ids"] + ) data_row["programmatic_finding_ids"] = "; ".join( data_row["programmatic_finding_ids"] ) + data_row["model_evidence"] = "; ".join( + data_row["model_evidence"] + ) + data_row["programmatic_evidence"] = "; ".join( + data_row["programmatic_evidence"] + ) writer.writerow(data_row) recall = score.workbook_row_recall programmatic = score.programmatic_coverage @@ -54,12 +106,14 @@ def write_reports(score: EvaluationScore, output_directory: Path) -> ReportPaths f"# Illustrative model evaluation: {score.model}", "", ( - "> This homepage result is illustrative and does not establish broad " + "> This homepage result is illustrative and does not establish " + "broad " "small-versus-large model equivalence." ), "", ( - f"- Workbook-row recall: {recall['caught']}/{recall['denominator']} " + "- Workbook-row recall: " + f"{recall['caught']}/{recall['denominator']} " f"({_percentage(recall)})" ), ( @@ -72,19 +126,45 @@ def write_reports(score: EvaluationScore, output_directory: Path) -> ReportPaths ), f"- Estimated audit cost: ${score.audit_cost_usd}", f"- Estimated evaluation cost: ${score.evaluation_cost_usd}", - f"- Estimated total experiment cost: ${score.total_experiment_cost_usd}", + ( + "- Estimated total experiment cost: " + f"${score.total_experiment_cost_usd}" + ), + f"- Input tokens: {score.usage['input_tokens']}", + f"- Cached input tokens: {score.usage['cached_input_tokens']}", + f"- Output tokens: {score.usage['output_tokens']}", + f"- Reasoning tokens: {score.usage['reasoning_tokens']}", + ( + "- Cache-creation input tokens: " + f"{score.usage['cache_creation_input_tokens']}" + ), + f"- End-to-end wall time: {score.wall_time_seconds:g}s", + f"- Request duration sum: {score.request_duration_sum_seconds:g}s", + "- Per-request durations: " + + json.dumps(score.request_durations_seconds), f"- Complete and rankable: {score.rankable}", f"- Format failures: {len(score.parse_failures)}", - f"- Unmatched audit findings: {len(score.unmatched_audit_finding_ids)}", + ( + "- Unmatched audit findings: " + f"{len(score.unmatched_audit_finding_ids)}" + ), "", - "| Source row | Eligibility | Programmatic | Model | Disposition |", - "|---|---|---:|---:|---|", + ( + "| Source row | Problem | Location | Eligibility | Rationale | " + "Programmatic evidence | Model evidence | Disposition |" + ), + "|---|---|---|---|---|---|---|---|", ] for report_row in score.rows: lines.append( f"| {report_row.source_sheet}!{report_row.source_row} | " - f"{report_row.eligibility} | {report_row.programmatic_caught} | " - f"{report_row.model_caught} | {report_row.disposition} |" + f"{_markdown(report_row.problem)} | " + f"{_markdown(report_row.location)} | " + f"{report_row.eligibility} | " + f"{_markdown(report_row.eligibility_rationale)} | " + f"{_markdown('; '.join(report_row.programmatic_evidence))} | " + f"{_markdown('; '.join(report_row.model_evidence))} | " + f"{report_row.disposition} |" ) markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return ReportPaths(json=json_path, csv=csv_path, markdown=markdown_path) diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index b84eae1..c091a5d 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -120,3 +120,10 @@ class RunMetadata: usage: Usage = field(default_factory=Usage) wall_time_seconds: float = 0 request_durations_seconds: tuple[float, ...] = () + workbook_sha256: str = "" + snapshot_sha256: str = "" + reference_set_version: str = "" + prompt_hashes_identity: str = "" + parser_identity: str = "" + eligibility_identity: str = "" + configuration_identity: str = "" diff --git a/vision_aid/evaluation/scoring.py b/vision_aid/evaluation/scoring.py index 1d35c86..522f94f 100644 --- a/vision_aid/evaluation/scoring.py +++ b/vision_aid/evaluation/scoring.py @@ -18,7 +18,10 @@ def _metric(caught: int, denominator: int) -> dict[str, int | str]: - rate = Decimal(caught) / Decimal(denominator) if denominator else Decimal(0) + """Represent one exact count, denominator, and decimal rate.""" + rate = ( + Decimal(caught) / Decimal(denominator) if denominator else Decimal(0) + ) return {"caught": caught, "denominator": denominator, "rate": str(rate)} @@ -38,18 +41,21 @@ class RowScore: model_caught: bool model_finding_ids: tuple[str, ...] programmatic_finding_ids: tuple[str, ...] + model_evidence: tuple[str, ...] + programmatic_evidence: tuple[str, ...] disposition: str @dataclass(frozen=True) class EvaluationScore: - """The complete deterministic object from which every report is projected.""" + """Complete deterministic source object for every report projection.""" run_id: str model: str complete: bool rankable: bool comparable_identity: str + compatibility: dict[str, str] reference_set_version: str workbook_sha256: str workbook_row_recall: dict[str, int | str] @@ -61,6 +67,7 @@ class EvaluationScore: usage: dict[str, int] wall_time_seconds: float request_durations_seconds: tuple[float, ...] + request_duration_sum_seconds: float unmatched_audit_finding_ids: tuple[str, ...] parse_failures: tuple[str, ...] rows: tuple[RowScore, ...] @@ -74,6 +81,7 @@ def _caught_references( references: Sequence[ReferenceDefect], decisions: Sequence[MatchDecision], ) -> tuple[set[str], dict[str, tuple[str, ...]]]: + """Calculate binary row credit and its accepted finding evidence.""" accepted = [item for item in decisions if item.state == "accepted"] by_reference: dict[str, list[MatchDecision]] = {} for decision in accepted: @@ -113,8 +121,8 @@ def score_evaluation( ] if unapproved: raise ValueError( - "Scoring requires accepted eligibility decisions for every reference: " - + ", ".join(unapproved) + "Scoring requires accepted eligibility decisions for every " + "reference: " + ", ".join(unapproved) ) audit_matches = validate_match_decisions( reference_set.references, @@ -132,6 +140,10 @@ def score_evaluation( programmatic_caught, programmatic_evidence = _caught_references( reference_set.references, programmatic_matches ) + audit_finding_by_id = {item.finding_id: item for item in audit_findings} + programmatic_finding_by_id = { + item.finding_id: item for item in programmatic_findings + } llm_references = { item.reference_id for item in reference_set.references @@ -178,7 +190,21 @@ def score_evaluation( programmatic_caught=programmatic_hit, model_caught=model_hit, model_finding_ids=model_evidence[reference.reference_id], - programmatic_finding_ids=programmatic_evidence[reference.reference_id], + programmatic_finding_ids=programmatic_evidence[ + reference.reference_id + ], + model_evidence=tuple( + f"{audit_finding_by_id[finding_id].problem} @ " + f"{audit_finding_by_id[finding_id].location}" + for finding_id in model_evidence[reference.reference_id] + ), + programmatic_evidence=tuple( + f"{programmatic_finding_by_id[finding_id].problem} @ " + f"{programmatic_finding_by_id[finding_id].location}" + for finding_id in programmatic_evidence[ + reference.reference_id + ] + ), disposition=disposition, ) ) @@ -186,12 +212,37 @@ def score_evaluation( item.finding_id for item in audit_matches if item.state == "accepted" } total_cost = Decimal(run.audit_cost_usd) + Decimal(run.evaluation_cost_usd) + if ( + run.reference_set_version + and run.reference_set_version != reference_set.version + ): + raise ValueError( + "Run metadata reference-set version does not match the scored " + "input." + ) + if ( + run.workbook_sha256 + and run.workbook_sha256 != reference_set.workbook_sha256 + ): + raise ValueError( + "Run metadata workbook checksum does not match the scored input." + ) return EvaluationScore( run_id=run.run_id, model=run.model, complete=run.complete, rankable=run.complete, comparable_identity=run.comparable_identity, + compatibility={ + "benchmark": run.comparable_identity, + "workbook": reference_set.workbook_sha256, + "snapshot": run.snapshot_sha256, + "reference_set": reference_set.version, + "prompts": run.prompt_hashes_identity, + "parser": run.parser_identity, + "eligibility": run.eligibility_identity, + "configuration": run.configuration_identity, + }, reference_set_version=reference_set.version, workbook_sha256=reference_set.workbook_sha256, workbook_row_recall=_metric( @@ -210,6 +261,7 @@ def score_evaluation( usage=asdict(run.usage), wall_time_seconds=run.wall_time_seconds, request_durations_seconds=run.request_durations_seconds, + request_duration_sum_seconds=sum(run.request_durations_seconds), unmatched_audit_finding_ids=tuple( sorted( item.finding_id @@ -222,14 +274,19 @@ def score_evaluation( ) -def rank_scores(scores: Sequence[EvaluationScore]) -> tuple[EvaluationScore, ...]: - """Rank comparable complete runs by recall, then unrounded audit cost only.""" +def rank_scores( + scores: Sequence[EvaluationScore], +) -> tuple[EvaluationScore, ...]: + """Rank complete runs by recall, then unrounded audit cost only.""" if any(not item.rankable for item in scores): - raise ValueError("An incomplete run is unranked and cannot be compared.") - identities = {item.comparable_identity for item in scores} + raise ValueError( + "An incomplete run is unranked and cannot be compared." + ) + identities = {tuple(sorted(item.compatibility.items())) for item in scores} if len(identities) > 1: raise ValueError( - "Runs use incompatible benchmark versions; cross-version analysis must " + "Runs use incompatible benchmark versions; cross-version analysis " + "must " "be explicitly requested and labeled." ) return tuple( @@ -238,7 +295,6 @@ def rank_scores(scores: Sequence[EvaluationScore]) -> tuple[EvaluationScore, ... key=lambda item: ( -Decimal(str(item.workbook_row_recall["rate"])), Decimal(item.audit_cost_usd), - item.run_id, ), ) ) diff --git a/vision_aid/evaluation/serialization.py b/vision_aid/evaluation/serialization.py index 135fb92..4924be0 100644 --- a/vision_aid/evaluation/serialization.py +++ b/vision_aid/evaluation/serialization.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from dataclasses import asdict from pathlib import Path @@ -19,6 +20,7 @@ def write_json(path: Path, value: object) -> None: + """Write stable, human-readable JSON to a private artifact path.""" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n", @@ -26,22 +28,40 @@ def write_json(path: Path, value: object) -> None: ) +def json_digest(value: object) -> str: + """Hash a canonical JSON representation for content identity checks.""" + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + def save_reference_set(path: Path, reference_set: ReferenceSet) -> None: + """Persist a canonical reference set.""" write_json(path, reference_set.to_dict()) def _provenance(value: dict[str, Any] | None) -> ReviewProvenance | None: + """Construct optional reviewer provenance from JSON data.""" return ReviewProvenance(**value) if value else None def load_reference_set(path: Path) -> ReferenceSet: + """Load a canonical reference set from a private artifact.""" data = json.loads(path.read_text(encoding="utf-8")) references = [] for value in data["references"]: value = dict(value) value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) - value["required_subdefects"] = tuple(value.get("required_subdefects", ())) - value["eligibility_review"] = _provenance(value.get("eligibility_review")) + value["required_subdefects"] = tuple( + value.get("required_subdefects", ()) + ) + value["eligibility_review"] = _provenance( + value.get("eligibility_review") + ) references.append(ReferenceDefect(**value)) return ReferenceSet( version=data["version"], @@ -53,10 +73,12 @@ def load_reference_set(path: Path) -> ReferenceSet: def save_findings(path: Path, findings: tuple[CanonicalFinding, ...]) -> None: + """Persist canonical findings without changing their raw evidence.""" write_json(path, [asdict(item) for item in findings]) def load_findings(path: Path) -> tuple[CanonicalFinding, ...]: + """Load canonical audit or programmatic findings.""" values = json.loads(path.read_text(encoding="utf-8")) findings = [] for value in values: @@ -65,11 +87,15 @@ def load_findings(path: Path) -> tuple[CanonicalFinding, ...]: return tuple(findings) -def save_match_decisions(path: Path, decisions: tuple[MatchDecision, ...]) -> None: +def save_match_decisions( + path: Path, decisions: tuple[MatchDecision, ...] +) -> None: + """Persist reviewed match decisions.""" write_json(path, [asdict(item) for item in decisions]) def load_match_decisions(path: Path) -> tuple[MatchDecision, ...]: + """Load reviewed match decisions and their provenance.""" values = json.loads(path.read_text(encoding="utf-8")) decisions = [] for value in values: @@ -79,9 +105,84 @@ def load_match_decisions(path: Path) -> tuple[MatchDecision, ...]: def load_run_metadata(value: dict[str, Any]) -> RunMetadata: + """Load synthetic run metadata from an in-memory test bundle.""" value = dict(value) value["usage"] = Usage(**value.get("usage", {})) value["request_durations_seconds"] = tuple( value.get("request_durations_seconds", ()) ) return RunMetadata(**value) + + +def load_verified_run_metadata( + path: Path, + reference_set: ReferenceSet, +) -> RunMetadata: + """Derive score metadata from a manifest and verify every identity.""" + manifest = json.loads(path.read_text(encoding="utf-8")) + identity_fields = { + key: manifest[key] + for key in ( + "workbook_sha256", + "snapshot_sha256", + "reference_set_version", + "eligibility_identity", + "configuration", + "filters", + "prompt_hashes", + "parser_hash", + "pricing_identity", + ) + } + benchmark_identity = json_digest(identity_fields) + if manifest.get("benchmark_identity") != benchmark_identity: + raise ValueError( + "Run manifest benchmark identities are inconsistent or changed." + ) + if manifest["workbook_sha256"] != reference_set.workbook_sha256: + raise ValueError( + "Run manifest workbook checksum does not match references." + ) + if manifest["reference_set_version"] != reference_set.version: + raise ValueError( + "Run manifest reference-set version does not match references." + ) + usage_value = manifest.get("usage", {}) + usage = Usage( + **{ + key: int(usage_value.get(key, 0)) + for key in ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_tokens", + "cache_creation_input_tokens", + ) + } + ) + return RunMetadata( + run_id=str(manifest["run_id"]), + model=str(manifest["configuration"]["model"]), + complete=bool(manifest.get("complete", False)), + comparable_identity=benchmark_identity, + audit_cost_usd=str(manifest.get("estimated_audit_cost_usd", "0")), + evaluation_cost_usd=str(manifest.get("evaluation_cost_usd", "0")), + usage=usage, + wall_time_seconds=float(manifest.get("wall_time_seconds", 0)), + request_durations_seconds=tuple( + float(value) + for value in manifest.get("request_durations_seconds", ()) + ), + workbook_sha256=str(manifest["workbook_sha256"]), + snapshot_sha256=str(manifest["snapshot_sha256"]), + reference_set_version=str(manifest["reference_set_version"]), + prompt_hashes_identity=json_digest(manifest["prompt_hashes"]), + parser_identity=str(manifest["parser_hash"]), + eligibility_identity=str(manifest["eligibility_identity"]), + configuration_identity=json_digest( + { + "configuration": manifest["configuration"], + "filters": manifest["filters"], + } + ), + ) From 1cb479ad2ff9442a157ca899c24ff6eb398bf897 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 12:29:24 -0400 Subject: [PATCH 017/126] style(evaluation): wrap CLI fixture metadata --- tests/test_evaluation_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 8d4f282..5455049 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -22,7 +22,10 @@ SNAPSHOT_SHA256 = ( "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" ) -TEMPORAL_ASSUMPTION = "Stakeholders report that the homepage has not changed since the workbook audit." +TEMPORAL_ASSUMPTION = ( + "Stakeholders report that the homepage has not changed since the " + "workbook audit." +) class SnapshotHTTPServer(ThreadingHTTPServer): From 212961658868f3259a8ba2ac745a3311b038c3a7 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 16:53:29 -0400 Subject: [PATCH 018/126] fix(evaluation): bind reports to verified run inputs --- tests/test_evaluation_cli.py | 197 ++++++++++++++++--------- tests/test_scoring_and_reporting.py | 24 ++- vision_aid/evaluation/cli.py | 6 +- vision_aid/evaluation/schemas.py | 2 + vision_aid/evaluation/scoring.py | 27 ++++ vision_aid/evaluation/serialization.py | 12 ++ 6 files changed, 188 insertions(+), 80 deletions(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 5455049..aa4f4f6 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,18 +1,22 @@ -import json import hashlib +import json import os import subprocess import sys +from collections.abc import Callable, Iterator from contextlib import contextmanager from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from tempfile import TemporaryDirectory from threading import Thread -from typing import Callable, Iterator -import pytest import openpyxl +import pytest + +from vision_aid.evaluation import audit as evaluation_audit +from vision_aid.evaluation.audit import live_authorization_digest +from vision_aid.evaluation.cli import main as evaluation_main PROJECT_ROOT = Path(__file__).parents[1] SNAPSHOT_BODY = ( @@ -39,7 +43,7 @@ class SnapshotHTTPServer(ThreadingHTTPServer): class SnapshotHandler(BaseHTTPRequestHandler): """Return a byte-sensitive local response without external network access.""" - def do_GET(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + def do_GET(self) -> None: """Serve the synthetic homepage and record acquisition metadata.""" server = self.server assert isinstance(server, SnapshotHTTPServer) @@ -252,9 +256,7 @@ def test_prepare_captures_exact_html_with_snapshot_provenance( "request_user_agent": "Mozilla/5.0", "status_code": 200, } - retrieved_at = datetime.fromisoformat( - metadata["retrieved_at"].replace("Z", "+00:00") - ) + retrieved_at = datetime.fromisoformat(metadata["retrieved_at"]) assert retrieved_at.tzinfo is not None @@ -457,7 +459,11 @@ def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( assert "No network requests were made" in result.stdout -def test_cli_completes_the_synthetic_private_workflow(tmp_path: Path) -> None: +def test_cli_completes_the_synthetic_private_workflow( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: """Prepare, review, audit, and report compose without provider access.""" workbook_path = tmp_path / "synthetic.xlsx" workbook = openpyxl.Workbook() @@ -529,83 +535,111 @@ def test_cli_completes_the_synthetic_private_workflow(tmp_path: Path) -> None: approved_path = workspace / "references" / "approved.json" approved = json.loads(approved_path.read_text(encoding="utf-8")) - reference_id = approved["references"][0]["reference_id"] run_directory = workspace / "runs" / "dry-run" - finding_path = run_directory / "synthetic-audit-findings.json" - finding_path.write_text( - json.dumps( - [ - { - "finding_id": "finding-1", - "source": "audit", - "run_id": "synthetic-run", - "model": "gpt-5.6-luna", - "prompt": "informative_alt_quality", - "checklist": "CL03", - "page_url": "https://example.test/", - "problem_family": "informative_alt_quality", - "problem": "Hero alternative text is misleading", - "element": "img", - "location": "Hero image", - "wcag_evidence": ["1.1.1"], - "raw_source": {"issues": ["Misleading alt"]}, - "parse_status": "parsed", - } - ] - ), - encoding="utf-8", + plan_path = run_directory / "evaluation-manifest.json" + run_manifest = json.loads(plan_path.read_text(encoding="utf-8")) + + class SyntheticClient: + """Return deterministic successful responses without network access.""" + + def __init__(self, **_kwargs: object) -> None: + pass + + def call(self, _prompt: str) -> dict[str, object]: + return { + "success": True, + "response": json.dumps( + { + "problem": "Hero alternative text is misleading", + "element": "img", + "location": "Hero image", + "wcag": "1.1.1", + } + ), + "usage": {"input_tokens": 10, "output_tokens": 5}, + "duration_seconds": 0.01, + "stop_reason": "stop", + } + + maximum_cost = "0.01" + authorization = live_authorization_digest( + run_manifest["plan_digest"], maximum_cost ) - empty_findings = run_directory / "empty-findings.json" - empty_findings.write_text("[]", encoding="utf-8") - audit_matches = run_directory / "audit-matches.json" - audit_matches.write_text( - json.dumps( + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) + assert ( + evaluation_main( [ - { - "reference_id": reference_id, - "finding_id": "finding-1", - "state": "accepted", - "provenance": { - "reviewer": "reviewer@example.test", - "rationale": "Same failure and hero image.", - "confidence": 0.9, - "timestamp": "2026-09-19T12:30:00Z", - }, - "required_subdefect": None, - "page_compatible": True, - "failure_compatible": True, - "location_compatible": True, - "notes": "", - } + "audit", + "--live", + "--plan", + str(plan_path), + "--authorize", + authorization, + "--max-audit-cost-usd", + maximum_cost, ] - ), - encoding="utf-8", + ) + == 0 ) - empty_matches = run_directory / "empty-matches.json" - empty_matches.write_text("[]", encoding="utf-8") - plan_path = run_directory / "evaluation-manifest.json" - run_manifest = json.loads(plan_path.read_text(encoding="utf-8")) - run_manifest.update( - { - "complete": True, - "estimated_audit_cost_usd": "0.001", - "usage": {"input_tokens": 10, "output_tokens": 5}, - "wall_time_seconds": 1.5, - "request_durations_seconds": [1.0], + capsys.readouterr() + + finding_path = run_directory / "canonical-audit-findings.json" + matches_workbook = workspace / "reviews" / "matches.xlsx" + generated = run_cli( + "review", + "--reference-set", + str(approved_path), + "--findings", + str(finding_path), + "--matches-workbook", + str(matches_workbook), + cwd=tmp_path, + ) + assert generated.returncode == 0, generated.stderr + match_review = openpyxl.load_workbook(matches_workbook) + matches = match_review["Matches"] + match_columns = {cell.value: cell.column for cell in matches[1]} + for row in range(2, matches.max_row + 1): + values = { + "decision": "accepted" if row == 2 else "rejected", + "page_compatible": True, + "failure_compatible": True, + "location_compatible": True, + "rationale": "Same reviewed homepage evidence.", + "reviewer": "reviewer@example.test", + "confidence": 0.9, + "timestamp": "2026-09-19T12:30:00Z", } + for name, value in values.items(): + matches.cell(row, match_columns[name], value) + match_review.save(matches_workbook) + imported = run_cli( + "review", + "--reference-set", + str(approved_path), + "--findings", + str(finding_path), + "--matches-workbook", + str(matches_workbook), + "--import-matches", + cwd=tmp_path, ) - synthetic_manifest = run_directory / "synthetic-live-manifest.json" - synthetic_manifest.write_text(json.dumps(run_manifest), encoding="utf-8") + assert imported.returncode == 0, imported.stderr + + empty_matches = run_directory / "empty-matches.json" + empty_matches.write_text("[]", encoding="utf-8") bundle_path = workspace / "synthetic-bundle.json" bundle_path.write_text( json.dumps( { "reference_set": "references/approved.json", - "audit_findings": "runs/dry-run/synthetic-audit-findings.json", - "programmatic_findings": "runs/dry-run/empty-findings.json", - "audit_matches": "runs/dry-run/audit-matches.json", + "audit_findings": "runs/dry-run/canonical-audit-findings.json", + "programmatic_findings": "runs/dry-run/canonical-programmatic.json", + "audit_matches": "reviews/match-decisions.json", "programmatic_matches": "runs/dry-run/empty-matches.json", - "run_manifest": "runs/dry-run/synthetic-live-manifest.json", + "run_manifest": "runs/dry-run/live-manifest.json", } ), encoding="utf-8", @@ -615,3 +649,22 @@ def test_cli_completes_the_synthetic_private_workflow(tmp_path: Path) -> None: assert reported.returncode == 0, reported.stderr assert (workspace / "reports" / f"{run_manifest['run_id']}.json").is_file() + + findings = json.loads(finding_path.read_text(encoding="utf-8")) + findings[0]["run_id"] = "different-run" + finding_path.write_text(json.dumps(findings), encoding="utf-8") + rejected_finding = run_cli( + "report", "--bundle", str(bundle_path), cwd=tmp_path + ) + assert rejected_finding.returncode == 2 + assert "do not belong to the verified run" in rejected_finding.stderr + + findings[0]["run_id"] = run_manifest["run_id"] + finding_path.write_text(json.dumps(findings), encoding="utf-8") + approved["references"][0]["eligibility_review"]["rationale"] = "Changed" + approved_path.write_text(json.dumps(approved), encoding="utf-8") + rejected_eligibility = run_cli( + "report", "--bundle", str(bundle_path), cwd=tmp_path + ) + assert rejected_eligibility.returncode == 2 + assert "eligibility decisions" in rejected_eligibility.stderr diff --git a/tests/test_scoring_and_reporting.py b/tests/test_scoring_and_reporting.py index 5710282..e473912 100644 --- a/tests/test_scoring_and_reporting.py +++ b/tests/test_scoring_and_reporting.py @@ -288,12 +288,22 @@ def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: eligibility_identity="eligibility-v1", configuration_identity="configuration-v1", ) + expensive_finding = replace( + finding, + run_id=base_run.run_id, + model=base_run.model, + ) expensive = score_evaluation( - references, (finding,), (), (_match("ref1", "finding1"),), (), base_run + references, + (expensive_finding,), + (), + (_match("ref1", "finding1"),), + (), + base_run, ) cheap = score_evaluation( references, - (replace(finding, run_id="cheap"),), + (replace(finding, run_id="cheap", model="model-cheap"),), (), (_match("ref1", "finding1"),), (), @@ -313,7 +323,7 @@ def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: incompatible = score_evaluation( replace(references, version="v2"), - (finding,), + (replace(finding, run_id="incompatible", model="other"),), (), (_match("ref1", "finding1"),), (), @@ -342,7 +352,13 @@ def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: ) changed = score_evaluation( references, - (finding,), + ( + replace( + finding, + run_id=changed_run.run_id, + model=changed_run.model, + ), + ), (), (_match("ref1", "finding1"),), (), diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 24a0251..f40de60 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -1,7 +1,6 @@ """Command-line shell for the private model-evaluation lifecycle.""" import argparse -import hashlib import json import os from collections.abc import Callable, Sequence @@ -25,6 +24,7 @@ load_match_decisions, load_reference_set, load_verified_run_metadata, + reference_set_identity, save_match_decisions, save_reference_set, ) @@ -255,9 +255,7 @@ def _audit(arguments: argparse.Namespace) -> int: snapshot_metadata = json.loads( metadata_path.read_text(encoding="utf-8") ) - eligibility_identity = hashlib.sha256( - reference_path.read_bytes() - ).hexdigest() + eligibility_identity = reference_set_identity(reference_set) run_directory = ( arguments.run_dir or workspace.root / "runs" / "dry-run" ) diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index c091a5d..85e956e 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -127,3 +127,5 @@ class RunMetadata: parser_identity: str = "" eligibility_identity: str = "" configuration_identity: str = "" + homepage_url: str = "" + prompt_names: tuple[str, ...] = () diff --git a/vision_aid/evaluation/scoring.py b/vision_aid/evaluation/scoring.py index 522f94f..79632df 100644 --- a/vision_aid/evaluation/scoring.py +++ b/vision_aid/evaluation/scoring.py @@ -114,6 +114,33 @@ def score_evaluation( parse_failures: Sequence[str] = (), ) -> EvaluationScore: """Calculate binary workbook-row credit from approved normalized inputs.""" + invalid_audit_findings = [ + item.finding_id + for item in audit_findings + if item.source != "audit" + or item.run_id != run.run_id + or item.model != run.model + or (run.homepage_url and item.page_url != run.homepage_url) + or (run.prompt_names and item.prompt not in run.prompt_names) + ] + if invalid_audit_findings: + raise ValueError( + "Audit findings do not belong to the verified run: " + + ", ".join(invalid_audit_findings) + ) + invalid_programmatic_findings = [ + item.finding_id + for item in programmatic_findings + if item.source != "programmatic" + or item.run_id != run.run_id + or item.model is not None + or (run.homepage_url and item.page_url != run.homepage_url) + ] + if invalid_programmatic_findings: + raise ValueError( + "Programmatic findings do not belong to the verified run: " + + ", ".join(invalid_programmatic_findings) + ) unapproved = [ item.reference_id for item in reference_set.references diff --git a/vision_aid/evaluation/serialization.py b/vision_aid/evaluation/serialization.py index 4924be0..aeb1aae 100644 --- a/vision_aid/evaluation/serialization.py +++ b/vision_aid/evaluation/serialization.py @@ -39,6 +39,11 @@ def json_digest(value: object) -> str: return hashlib.sha256(encoded).hexdigest() +def reference_set_identity(reference_set: ReferenceSet) -> str: + """Hash all approved reference and eligibility decision content.""" + return json_digest(reference_set.to_dict()) + + def save_reference_set(path: Path, reference_set: ReferenceSet) -> None: """Persist a canonical reference set.""" write_json(path, reference_set.to_dict()) @@ -147,6 +152,11 @@ def load_verified_run_metadata( raise ValueError( "Run manifest reference-set version does not match references." ) + expected_eligibility = reference_set_identity(reference_set) + if manifest["eligibility_identity"] != expected_eligibility: + raise ValueError( + "Run manifest eligibility decisions do not match references." + ) usage_value = manifest.get("usage", {}) usage = Usage( **{ @@ -185,4 +195,6 @@ def load_verified_run_metadata( "filters": manifest["filters"], } ), + homepage_url=str(manifest["homepage_url"]), + prompt_names=tuple(item["name"] for item in manifest["prompts"]), ) From 68d34d9cc228a10251f35f42177108f07ebee04b Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 16:53:59 -0400 Subject: [PATCH 019/126] docs(evaluation): resolve framework implementation tickets --- .../issues/04-homepage-reference-set.md | 24 ++++++++++++------- ...5-review-and-score-synthetic-comparison.md | 24 ++++++++++++------- .../06-programmatic-workbook-coverage.md | 24 ++++++++++++------- .../issues/07-safe-luna-dry-run.md | 22 ++++++++++------- .../issues/08-authorized-live-audit.md | 24 ++++++++++++------- .../issues/09-normalize-raw-audit-findings.md | 22 ++++++++++------- .../issues/10-review-finding-matches.md | 24 ++++++++++++------- .../11-verify-complete-evaluation-workflow.md | 24 ++++++++++++------- 8 files changed, 118 insertions(+), 70 deletions(-) diff --git a/.scratch/model-evaluation/issues/04-homepage-reference-set.md b/.scratch/model-evaluation/issues/04-homepage-reference-set.md index 5ea2ecb..f3f6ff0 100644 --- a/.scratch/model-evaluation/issues/04-homepage-reference-set.md +++ b/.scratch/model-evaluation/issues/04-homepage-reference-set.md @@ -4,13 +4,19 @@ **Blocked by:** 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot. -**Status:** ready-for-agent +**Status:** resolved -- [ ] Workbook absence and SHA-256 drift fail with clear acquisition or validation guidance. -- [ ] Imported records retain workbook filename, sheet, source row, raw evidence, canonical URL, page scope, and WCAG evidence without semantic rewriting. -- [ ] The reference set includes every homepage and applicable Global row exactly once. -- [ ] Eligibility is limited to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with rationale and approval state required. -- [ ] Global rows are represented as global claims with homepage evidence rather than multiplied across pages. -- [ ] A private Eligibility workbook presents source evidence, proposed classification, rationale, approval, reviewer identity, confidence, timestamp, and notes. -- [ ] Human-reviewed decisions round-trip into a validated, versioned reference set through the `EligibilityReviewer` interface. -- [ ] Synthetic workbook tests cover provenance, filtering, review import, invalid decisions, and checksum validation. +- [x] Workbook absence and SHA-256 drift fail with clear acquisition or validation guidance. +- [x] Imported records retain workbook filename, sheet, source row, raw evidence, canonical URL, page scope, and WCAG evidence without semantic rewriting. +- [x] The reference set includes every homepage and applicable Global row exactly once. +- [x] Eligibility is limited to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with rationale and approval state required. +- [x] Global rows are represented as global claims with homepage evidence rather than multiplied across pages. +- [x] A private Eligibility workbook presents source evidence, proposed classification, rationale, approval, reviewer identity, confidence, timestamp, and notes. +- [x] Human-reviewed decisions round-trip into a validated, versioned reference set through the `EligibilityReviewer` interface. +- [x] Synthetic workbook tests cover provenance, filtering, review import, invalid decisions, and checksum validation. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Verified by the synthetic reference-set and CLI tests within the +55-test passing suite. diff --git a/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md b/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md index 560ce53..e4023c3 100644 --- a/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md +++ b/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md @@ -4,13 +4,19 @@ **Blocked by:** 04: Generate and validate the homepage reference set. -**Status:** ready-for-agent +**Status:** resolved -- [ ] The `MatchReviewer` interface accepts human-reviewed decisions without coupling the scorer to Excel or future model adapters. -- [ ] Match decisions support accepted, rejected, and needs-review states with reviewer provenance, rationale, confidence, and timestamp. -- [ ] Validation enforces one-to-one matches and rejects conflicting accepted decisions. -- [ ] Grouped rows receive one point when their grouped defect is matched, while rows with distinct required sub-defects require all parts. -- [ ] The deterministic scorer calculates workbook-row recall, programmatic coverage, combined workbook coverage, unmatched findings, and parse failures from approved inputs. -- [ ] Estimated audit cost is the sole ranking tie-breaker; latency, token usage, and evaluation cost remain informational. -- [ ] JSON, CSV, and Markdown reports derive from one score object and agree on every total and row disposition. -- [ ] Repeated scoring of identical inputs produces byte-equivalent structured results apart from deliberately excluded presentation timestamps. +- [x] The `MatchReviewer` interface accepts human-reviewed decisions without coupling the scorer to Excel or future model adapters. +- [x] Match decisions support accepted, rejected, and needs-review states with reviewer provenance, rationale, confidence, and timestamp. +- [x] Validation enforces one-to-one matches and rejects conflicting accepted decisions. +- [x] Grouped rows receive one point when their grouped defect is matched, while rows with distinct required sub-defects require all parts. +- [x] The deterministic scorer calculates workbook-row recall, programmatic coverage, combined workbook coverage, unmatched findings, and parse failures from approved inputs. +- [x] Estimated audit cost is the sole ranking tie-breaker; latency, token usage, and evaluation cost remain informational. +- [x] JSON, CSV, and Markdown reports derive from one score object and agree on every total and row disposition. +- [x] Repeated scoring of identical inputs produces byte-equivalent structured results apart from deliberately excluded presentation timestamps. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Verified by deterministic scoring, matching, ranking, and +cross-format report tests within the 55-test passing suite. diff --git a/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md b/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md index 396223a..ebc0310 100644 --- a/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md +++ b/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md @@ -4,12 +4,18 @@ **Blocked by:** 04: Generate and validate the homepage reference set; 05: Review and score a synthetic comparison. -**Status:** ready-for-agent - -- [ ] Existing semantic, form, and non-text programmatic checks run against the frozen snapshot without duplicated audit logic. -- [ ] Programmatic findings are normalized with stable page and location evidence while retaining their raw source. -- [ ] Reference matches are reviewable and use the same accepted-decision validation as other matches. -- [ ] A `programmatic` eligibility classification does not automatically award credit when the checker misses the row. -- [ ] Reports identify covered programmatic rows, uncovered checker responsibilities, unavailable evidence, and ambiguity. -- [ ] Programmatic results contribute to combined workbook coverage but never to model ranking. -- [ ] Tests exercise the complete programmatic path with synthetic HTML and approved match decisions. +**Status:** resolved + +- [x] Existing semantic, form, and non-text programmatic checks run against the frozen snapshot without duplicated audit logic. +- [x] Programmatic findings are normalized with stable page and location evidence while retaining their raw source. +- [x] Reference matches are reviewable and use the same accepted-decision validation as other matches. +- [x] A `programmatic` eligibility classification does not automatically award credit when the checker misses the row. +- [x] Reports identify covered programmatic rows, uncovered checker responsibilities, unavailable evidence, and ambiguity. +- [x] Programmatic results contribute to combined workbook coverage but never to model ranking. +- [x] Tests exercise the complete programmatic path with synthetic HTML and approved match decisions. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Verified with synthetic HTML, approved match decisions, and the +production pipeline dry run. diff --git a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md index 49c1252..e846876 100644 --- a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md +++ b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md @@ -4,11 +4,17 @@ **Blocked by:** 01: Make audit request configuration explicit; 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot; 04: Generate and validate the homepage reference set. -**Status:** ready-for-agent - -- [ ] The dry run uses the frozen prompts, filters, prompt order, disabled summaries, sequential execution, and existing output limit. -- [ ] The planned configuration records GPT-5.6 Luna, medium reasoning, Chat Completions, and omitted temperature. -- [ ] Prompt payloads, request count, estimated input tokens, prompt hashes, parser hashes, and repository state are saved without making a provider call. -- [ ] The manifest records workbook, snapshot, reference-set, benchmark, configuration, and pricing identities. -- [ ] The command refuses live execution when the live flag, API key, approved reference set, or maximum audit-cost limit is absent. -- [ ] Tests prove that an environment key alone cannot trigger spending. +**Status:** resolved + +- [x] The dry run uses the frozen prompts, filters, prompt order, disabled summaries, sequential execution, and existing output limit. +- [x] The planned configuration records GPT-5.6 Luna, medium reasoning, Chat Completions, and omitted temperature. +- [x] Prompt payloads, request count, estimated input tokens, prompt hashes, parser hashes, and repository state are saved without making a provider call. +- [x] The manifest records workbook, snapshot, reference-set, benchmark, configuration, and pricing identities. +- [x] The command refuses live execution when the live flag, API key, approved reference set, or maximum audit-cost limit is absent. +- [x] Tests prove that an environment key alone cannot trigger spending. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Verified by dry-run manifest and no-spend gate tests; no provider +call is made by the test suite. diff --git a/.scratch/model-evaluation/issues/08-authorized-live-audit.md b/.scratch/model-evaluation/issues/08-authorized-live-audit.md index 144f88c..24c92ec 100644 --- a/.scratch/model-evaluation/issues/08-authorized-live-audit.md +++ b/.scratch/model-evaluation/issues/08-authorized-live-audit.md @@ -4,13 +4,19 @@ **Blocked by:** 01: Make audit request configuration explicit; 07: Prepare a safe Luna dry run. -**Status:** ready-for-agent +**Status:** resolved -- [ ] Live execution requires the explicit live flag, API key, maximum audit-cost limit, and the exact summarized benchmark configuration. -- [ ] Audit prompts run sequentially with medium reasoning, no temperature, disabled summaries, and the existing output limit. -- [ ] At most two additional attempts occur for timeouts, rate limits, and server errors using bounded exponential backoff. -- [ ] Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. -- [ ] Every attempt records request identity, response metadata, usage, duration, stop reason, and failure classification without exposing secrets. -- [ ] Billed retry usage contributes to audit cost, and the runner stops before starting another request once the cost guardrail is exhausted. -- [ ] Exhausted transient failures mark the run incomplete and unranked; malformed successful responses remain recorded as model-format failures. -- [ ] Fake-provider tests exercise success, retry, budget, malformed response, and incomplete-run behavior without spending money. +- [x] Live execution requires the explicit live flag, API key, maximum audit-cost limit, and the exact summarized benchmark configuration. +- [x] Audit prompts run sequentially with medium reasoning, no temperature, disabled summaries, and the existing output limit. +- [x] At most two additional attempts occur for timeouts, rate limits, and server errors using bounded exponential backoff. +- [x] Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. +- [x] Every attempt records request identity, response metadata, usage, duration, stop reason, and failure classification without exposing secrets. +- [x] Billed retry usage contributes to audit cost, and the runner stops before starting another request once the cost guardrail is exhausted. +- [x] Exhausted transient failures mark the run incomplete and unranked; malformed successful responses remain recorded as model-format failures. +- [x] Fake-provider tests exercise success, retry, budget, malformed response, and incomplete-run behavior without spending money. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Fake-provider tests verify authorization, retry, checkpoint, +budget, malformed-response, and incomplete-run policies without spending. diff --git a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md index 6295696..eba1aa6 100644 --- a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md +++ b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md @@ -4,11 +4,17 @@ **Blocked by:** 08: Execute and record an authorized live audit. -**Status:** ready-for-agent - -- [ ] Every active non-summary prompt has a fixed normalizer, including categories omitted by the production CSV. -- [ ] Canonical audit findings retain run and model identity, prompt, checklist, page, problem family and statement, location evidence, WCAG evidence, raw source, and parse status. -- [ ] Valid object, array, fenced JSON, empty, malformed, and unknown prompt responses produce explicit deterministic outcomes. -- [ ] Successful malformed responses produce no usable findings and remain visible as format failures. -- [ ] Normalization performs no model or network call and applies no heuristic suppression or deduplication. -- [ ] Synthetic fixtures cover every prompt normalizer without copying private or live provider data. +**Status:** resolved + +- [x] Every active non-summary prompt has a fixed normalizer, including categories omitted by the production CSV. +- [x] Canonical audit findings retain run and model identity, prompt, checklist, page, problem family and statement, location evidence, WCAG evidence, raw source, and parse status. +- [x] Valid object, array, fenced JSON, empty, malformed, and unknown prompt responses produce explicit deterministic outcomes. +- [x] Successful malformed responses produce no usable findings and remain visible as format failures. +- [x] Normalization performs no model or network call and applies no heuristic suppression or deduplication. +- [x] Synthetic fixtures cover every prompt normalizer without copying private or live provider data. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Synthetic normalization tests cover the complete prompt registry +and all specified parse outcomes. diff --git a/.scratch/model-evaluation/issues/10-review-finding-matches.md b/.scratch/model-evaluation/issues/10-review-finding-matches.md index 0c352e7..8551883 100644 --- a/.scratch/model-evaluation/issues/10-review-finding-matches.md +++ b/.scratch/model-evaluation/issues/10-review-finding-matches.md @@ -4,12 +4,18 @@ **Blocked by:** 04: Generate and validate the homepage reference set; 09: Normalize every raw audit prompt response. -**Status:** ready-for-agent - -- [ ] Candidate generation removes only hard incompatibilities such as different pages or incompatible evidence categories. -- [ ] Candidate generation never awards credit or makes a semantic match decision. -- [ ] A private Matches workbook presents reference evidence, audit evidence, proposed disposition, rationale, confidence, approval, reviewer identity, timestamp, and notes side by side. -- [ ] An accepted match requires compatible page or scope, the same underlying accessibility failure, and compatible location, element, or element group. -- [ ] WCAG identifiers and remediation wording remain supporting evidence rather than exact-match requirements. -- [ ] Approved decisions round-trip through the `MatchReviewer` interface and enforce one-to-one matching. -- [ ] Conflicts, unresolved decisions, incomplete grouped evidence, and missing required sub-defects fail clearly rather than being resolved silently. +**Status:** resolved + +- [x] Candidate generation removes only hard incompatibilities such as different pages or incompatible evidence categories. +- [x] Candidate generation never awards credit or makes a semantic match decision. +- [x] A private Matches workbook presents reference evidence, audit evidence, proposed disposition, rationale, confidence, approval, reviewer identity, timestamp, and notes side by side. +- [x] An accepted match requires compatible page or scope, the same underlying accessibility failure, and compatible location, element, or element group. +- [x] WCAG identifiers and remediation wording remain supporting evidence rather than exact-match requirements. +- [x] Approved decisions round-trip through the `MatchReviewer` interface and enforce one-to-one matching. +- [x] Conflicts, unresolved decisions, incomplete grouped evidence, and missing required sub-defects fail clearly rather than being resolved silently. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. Candidate generation, workbook round-trip, evidence validation, +and conflict tests pass in the synthetic suite. diff --git a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md index 68885a4..91d498b 100644 --- a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md +++ b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md @@ -4,13 +4,19 @@ **Blocked by:** 05: Review and score a synthetic comparison; 06: Measure programmatic workbook coverage; 08: Execute and record an authorized live audit; 10: Generate and validate finding matches. -**Status:** ready-for-agent +**Status:** resolved -- [ ] The CLI completes prepare, audit, review, and report workflows against a temporary private workspace using synthetic inputs. -- [ ] Ordinary comparisons reject mismatched snapshots, reference sets, prompts, parsers, eligibility decisions, or benchmark configurations. -- [ ] Run manifests contain content identity, code state, request configuration, retry policy, usage, pricing, timestamps, and run identity. -- [ ] Audit, evaluation, and total experiment costs remain separate, and only audit cost affects equal-recall ranking. -- [ ] End-to-end and per-request latency are reported but never ranked. -- [ ] JSON, CSV, and Markdown reports expose every in-scope workbook row and agree on scoring and coverage. -- [ ] CI runs the pytest suite without private workbook data, public network access, provider credentials, or spending. -- [ ] Documentation explains the private-data workflow, human approval gates, dry-run review, live authorization, and illustrative limits of the homepage benchmark. +- [x] The CLI completes prepare, audit, review, and report workflows against a temporary private workspace using synthetic inputs. +- [x] Ordinary comparisons reject mismatched snapshots, reference sets, prompts, parsers, eligibility decisions, or benchmark configurations. +- [x] Run manifests contain content identity, code state, request configuration, retry policy, usage, pricing, timestamps, and run identity. +- [x] Audit, evaluation, and total experiment costs remain separate, and only audit cost affects equal-recall ranking. +- [x] End-to-end and per-request latency are reported but never ranked. +- [x] JSON, CSV, and Markdown reports expose every in-scope workbook row and agree on scoring and coverage. +- [x] CI runs the pytest suite without private workbook data, public network access, provider credentials, or spending. +- [x] Documentation explains the private-data workflow, human approval gates, dry-run review, live authorization, and illustrative limits of the homepage benchmark. + +## Comments + +Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, +and identity-verified in `2129616`. The 55-test suite proves the synthetic CLI lifecycle and failure +policies; mypy, Ruff, lock validation, and the production dry run also pass. From 2448042942629d68558d7b03ba9d2192e24f6c94 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 18:30:07 -0400 Subject: [PATCH 020/126] feat(evaluation): streamline run-oriented CLI workflow --- .scratch/model-evaluation/spec.md | 14 +- CONTEXT.md | 8 + README.md | 69 ++++-- tests/test_evaluation_audit.py | 197 ++++++++++++--- tests/test_evaluation_cli.py | 385 +++++++++++++++++++++++++---- vision_aid/evaluation/audit.py | 288 ++++++++++++++------- vision_aid/evaluation/cli.py | 319 +++++++++++++++++------- vision_aid/evaluation/workspace.py | 11 + 8 files changed, 1009 insertions(+), 282 deletions(-) diff --git a/.scratch/model-evaluation/spec.md b/.scratch/model-evaluation/spec.md index 658932d..62c1224 100644 --- a/.scratch/model-evaluation/spec.md +++ b/.scratch/model-evaluation/spec.md @@ -12,7 +12,7 @@ Build a private, CLI-only model-evaluation framework around the existing audit p The deterministic scorer will remain separate from review. Human-reviewed files will initially implement the eligibility and match reviewer interfaces; future Astra-backed adapters can produce the same decisions without changing import, scoring, or reporting. Models will be ranked first by workbook-row recall over approved LLM-eligible rows and, only when recall is equal, by estimated audit cost. Programmatic coverage, combined workbook coverage, latency, token usage, malformed output, and unmatched findings will remain visible but will not alter model rank. -Private inputs and generated artifacts will stay outside version control. The committed repository will contain only the framework, schemas, documentation, versioned public pricing configuration, and synthetic tests. Live API execution will be impossible without an explicit live flag, a cost limit, an available key, and per-run authorization after a dry-run summary. +Private inputs and generated artifacts will stay outside version control. The committed repository will contain only the framework, schemas, documentation, versioned public pricing configuration, and synthetic tests. Live API execution will be impossible without an explicit live flag, a planned positive cost limit, an available key, and point-of-use live-run approval after the complete execution summary. ## User Stories @@ -70,7 +70,7 @@ Private inputs and generated artifacts will stay outside version control. The co 52. As a budget owner, I want the unrounded estimated audit cost used for ties, so that display rounding cannot change rank. 53. As an operator, I want dry-run behavior by default even when an API key exists, so that environment configuration alone cannot spend money. 54. As an operator, I want live execution to require an explicit flag, so that spending is intentional. -55. As an operator, I want each summarized proof-of-concept run explicitly authorized, so that approval covers a known model, snapshot, request set, and configuration. +55. As an operator, I want each summarized proof-of-concept run explicitly approved at execution time, so that approval covers a known model, snapshot, request set, configuration, and cost guardrail. 56. As an operator, I want a required maximum audit-cost guardrail, so that a live run stops before starting another request after exhausting its budget. 57. As an operator, I want bounded retries only for transient failures, so that recoverable provider errors do not automatically invalidate a run. 58. As an operator, I want every retry and any billed usage recorded, so that cost and reliability reports are complete. @@ -127,13 +127,13 @@ Private inputs and generated artifacts will stay outside version control. The co - A run manifest records workbook and snapshot hashes, reference-set version, source-control commit and dirty state, prompt and parser hashes, filters, summaries, model, reasoning effort, endpoint, output limit, retry policy, usage, price-schedule version, timestamps, and run ID. - Ordinary comparison rejects runs whose snapshot, reference set, prompts, or eligibility set differ. Cross-version analysis must be requested explicitly and labeled accordingly. - Reports are emitted as JSON, CSV, and Markdown from one score object. The row-level view exposes eligibility, rationale, programmatic outcome, model match, evidence, and final disposition for every in-scope workbook row. -- The proof of concept proceeds through gates: implementation and tests; snapshot and reference preparation; human eligibility approval; dry-run summary; explicit live-run authorization; Luna audit; human match approval; final report. -- One authorization covers the complete summarized live run, including policy-compliant automatic retries. A materially different configuration or separate rerun requires new authorization. +- The proof of concept proceeds through gates: implementation and tests; snapshot and reference preparation; human eligibility approval; evaluation-run planning; point-of-use live-run approval; Luna audit; human match approval; final report. +- One live-run approval covers the complete displayed execution summary, including policy-compliant automatic retries. A materially different configuration or separate rerun requires a newly planned evaluation run and new approval. - The homepage result is illustrative because the provisional inventory contains only three clearly LLM-eligible rows out of sixteen in-scope rows. Benchmark expansion is required before any small-versus-large model claim. ## Testing Decisions -- Tests assert external behavior at module interfaces, not private helper functions or implementation order. The principal test seam is the deterministic scorer's input bundle and returned score object because it exercises eligibility, one-to-one matching, ranking data, coverage, and report inputs at the highest stable seam. +- Tests assert external behavior at module interfaces, not private helper functions or implementation order. The principal scoring seam is the deterministic scorer's structured inputs and returned score object because it exercises eligibility, one-to-one matching, ranking data, coverage, and report inputs at the highest stable seam; the complete operator lifecycle is exercised through the evaluation CLI. - Workbook-import tests use synthetic Excel fixtures and assert provenance retention, homepage filtering, source-row identity, checksum validation, exact allowed eligibility states, and useful missing-file or drift errors. - Snapshot tests use local HTTP fakes or saved synthetic responses and assert immutable content, provenance fields, and hashing without accessing the live Pristine site. - Audit-adapter tests replace the production pipeline with a fake at its existing callable seam. They assert frozen configuration, raw-result preservation, manifest metadata, dry-run safety, live-flag requirements, cost-limit behavior, and retry classification without provider calls. @@ -166,7 +166,7 @@ Private inputs and generated artifacts will stay outside version control. The co - Treating Global rows as verified on every site page. - Expanding immediately to all seventeen workbook URLs or using repeated job-page rows to claim generalization. - Computing an exact invoiced dollar amount or accounting for private contractual discounts unavailable from API usage metadata. -- Making a live provider request before eligibility approval, dry-run review, explicit authorization, and a cost limit. +- Making a live provider request before eligibility approval, evaluation-run planning, point-of-use approval, and a positive cost limit. ## Further Notes @@ -174,5 +174,5 @@ Private inputs and generated artifacts will stay outside version control. The co - Provisional LLM-eligible examples include decorative-image alt judgment and heading-quality problems. Programmatic examples include missing landmarks, missing skip navigation, actionable empty alt, and multiple H1 elements. Unavailable-evidence examples include visual list semantics, styled text that should be headings, carousel controls, Lottie animation, decorative literal glyphs, and media operability. Ambiguities include fragmented adjacent links and a workbook alt-text claim that may conflict with the page. - The existing committed HTML fixture represents DAT Vision Aid, not Pristine, and cannot be used for this benchmark. - The workbook is the mandatory provenance source, but approved normalization may mark rows unavailable or ambiguous and may represent distinct required sub-defects. These transformations must remain traceable and reviewable. -- The first implementation milestone ends at the eligibility-review gate and incurs no API cost. The cost limit for the first live Luna run will be selected only after the dry-run summary exposes the request set and estimated usage. +- The first implementation milestone ends at the eligibility-review gate and incurs no API cost. The cost limit for the first live Luna run will be selected while planning the evaluation run, whose summary exposes the request set and estimated usage. - Future benchmark expansion should sample distinct pages and template families rather than random rows because repeated job-page patterns dominate the workbook. diff --git a/CONTEXT.md b/CONTEXT.md index 3f3e2f1..ccfa6a4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -24,6 +24,10 @@ _Avoid_: Every workbook row, programmatic defect An immutable capture of the audited webpage used as the common input for every evaluated model, with its source and capture identity recorded. _Avoid_: Live page, test URL +**Evaluation run**: +One planned model evaluation and its resulting audit evidence, reviewed match decisions, and reports, identified together for reproducibility. +_Avoid_: Dry-run folder, report bundle + **Eligibility classification**: The reviewed assignment explaining whether a reference defect belongs to the AI model's responsibility, a programmatic check, unavailable audit evidence, or unresolved ambiguity. _Avoid_: Model score, exclusion without rationale @@ -48,6 +52,10 @@ _Avoid_: Candidate match, similarity score The calculated cost of one audit run using API-reported token usage and a recorded model-price schedule. It is the sole tie-breaker when workbook-row recall is equal. _Avoid_: Invoice amount, latency score +**Live-run approval**: +An operator's explicit confirmation to execute a displayed audit plan under a maximum audit-cost guardrail. +_Avoid_: Authorization digest, API-key presence + **Comparison reviewer**: A human or model that proposes eligibility classifications or match decisions through the same review process. A comparison reviewer does not calculate the final score. _Avoid_: Scorer, audit model diff --git a/README.md b/README.md index d4eb7b9..537ddd8 100644 --- a/README.md +++ b/README.md @@ -302,29 +302,32 @@ workbook-row scoring unit supported by homepage evidence. `audit` is network-free by default even if `OPENAI_API_KEY` exists. With an approved reference set and benchmark snapshot it runs the production extractors, filters, programmatic checks, slicers, and unchanged prompts to -write an exact Luna dry-run plan: +create a uniquely identified evaluation run. Planning requires a positive +maximum audit cost and freezes that guardrail and the approved reference set +inside the run directory: ```bash uv run visionaid-evaluate audit \ --max-audit-cost-usd "$PROPOSED_LIMIT" ``` -Review the request count, prompt hashes, estimated input usage, model -configuration, pricing identity, and authorization digest in the printed -summary. The live authorization digest binds both the exact plan and proposed -cost limit. A billable run requires all of the following: `--live`, the exact -saved plan and live authorization digest, `OPENAI_API_KEY`, and that same -positive cost limit. The authorization is single-use and covers the summarized -run and its policy-compliant retries: +Keep the printed `.model-evaluation/runs/` path. The summary shows the +model, endpoint, reasoning effort, benchmark and reference identities, prompt +request set, estimated input usage, retry policy, saved cost guardrail, and +destination. A billable run requires `--live`, that run directory, and +`OPENAI_API_KEY`. The command revalidates the frozen evidence, prints the same +summary, and asks a terminal operator to type `yes`: ```bash uv run visionaid-evaluate audit \ --live \ - --plan .model-evaluation/runs/dry-run/evaluation-manifest.json \ - --authorize "$REVIEWED_AUTHORIZATION_DIGEST" \ - --max-audit-cost-usd "$APPROVED_LIMIT" + --run-dir .model-evaluation/runs/ ``` +Deliberate automation may add `--auto-approve`. It skips terminal input but +does not skip the summary, API credential, integrity checks, saved cost limit, +or overwrite protection. Any rerun requires planning a fresh evaluation run. + The runner uses `gpt-5.6-luna` through Chat Completions with medium reasoning, omits temperature, keeps summaries disabled, and preserves every attempt and raw response. It retries only transient timeout, rate-limit, and server @@ -332,12 +335,46 @@ failures (at most twice). Exhausted failures make a run incomplete and unrankable; successful malformed JSON is retained as a format failure and is not retried. -Canonical audit findings and programmatic findings are reviewed through a -private Matches workbook. Accepted matches require compatible page scope, +Canonical audit findings and programmatic findings use separate, run-scoped +Matches workbooks. Generate, complete, and import both review kinds: + +```bash +uv run visionaid-evaluate review \ + --run-dir .model-evaluation/runs/ \ + --kind audit +uv run visionaid-evaluate review \ + --run-dir .model-evaluation/runs/ \ + --kind audit \ + --import-matches +uv run visionaid-evaluate review \ + --run-dir .model-evaluation/runs/ \ + --kind programmatic +uv run visionaid-evaluate review \ + --run-dir .model-evaluation/runs/ \ + --kind programmatic \ + --import-matches +``` + +The conventional workbooks and decision files live under +`.model-evaluation/reviews//`. Explicit `--reference-set`, `--findings`, +`--matches-workbook`, and `--match-decisions` overrides remain available for +investigation and recovery. Accepted matches require compatible page scope, underlying failure, and location or element-group evidence; WCAG labels and -remediation wording are supporting evidence. The scorer is network-free, -enforces one-to-one matching and all required sub-defects, and emits JSON, CSV, -and Markdown from one score object. +remediation wording are supporting evidence. + +After both reviews, reporting discovers the frozen references, both finding +collections, both decision files, the live manifest, and format failures from +the evaluation run: + +```bash +uv run visionaid-evaluate report \ + --run-dir .model-evaluation/runs/ +``` + +The scorer is network-free, enforces one-to-one matching and all required +sub-defects, and emits JSON, CSV, and Markdown from one score object. Missing +review artifacts produce the exact next review command instead of requiring a +hand-written report bundle. Workbook-row recall over approved LLM-eligible rows is the primary ranking metric. Unrounded estimated audit cost is the only tie-breaker. Programmatic diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py index 8f59fe9..24a7fa4 100644 --- a/tests/test_evaluation_audit.py +++ b/tests/test_evaluation_audit.py @@ -7,10 +7,31 @@ from vision_aid.evaluation.audit import ( AuditExecutionError, build_audit_plan, - execute_authorized_audit, - live_authorization_digest, + execute_audit_run, ) from vision_aid.evaluation.pricing import PriceSchedule +from vision_aid.evaluation.schemas import ReferenceSet +from vision_aid.evaluation.serialization import ( + reference_set_identity, + save_reference_set, +) + +REFERENCE_SET = ReferenceSet( + version="references-v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=(), +) +REFERENCE_IDENTITY = reference_set_identity(REFERENCE_SET) +APPROVED_AT = "2026-09-19T12:00:00Z" + + +def _reference_set_path(directory: Path) -> Path: + """Write the frozen synthetic reference set used by an audit plan.""" + path = directory / "reference-set.json" + save_reference_set(path, REFERENCE_SET) + return path def _fake_pipeline( @@ -90,10 +111,12 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( plan = build_audit_plan( snapshot, run_directory, + maximum_audit_cost_usd="1.00", + reference_set_path=_reference_set_path(run_directory), workbook_sha256="a" * 64, snapshot_sha256=snapshot_sha256, reference_set_version="references-v1", - eligibility_identity="c" * 64, + eligibility_identity=REFERENCE_IDENTITY, pipeline=_fake_pipeline, repository=tmp_path, ) @@ -109,6 +132,7 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( } assert plan["request_count"] == 2 assert plan["estimated_input_tokens"] == 30 + assert plan["maximum_audit_cost_usd"] == "1" assert list(plan["prompt_hashes"]) == ["heading_structure", "link_clarity"] assert len(plan["parser_hash"]) == 64 assert len(plan["pricing_identity"]) == 64 @@ -116,6 +140,31 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( assert (run_directory / "evaluation-manifest.json").is_file() +@pytest.mark.parametrize("limit", ("0", "-1", "NaN", "not-a-number")) +def test_audit_plan_requires_a_positive_finite_cost_limit( + tmp_path: Path, + limit: str, +) -> None: + """Planning rejects absent economic meaning before running the pipeline.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "run" + + with pytest.raises(AuditExecutionError, match="cost limit"): + build_audit_plan( + snapshot, + run_directory, + maximum_audit_cost_usd=limit, + reference_set_path=_reference_set_path(run_directory), + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity=REFERENCE_IDENTITY, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + + class FakeClient: def __init__(self, outcomes: list[dict]): """Queue provider outcomes in request order.""" @@ -196,33 +245,33 @@ def test_live_gates_retries_and_format_failures( plan = build_audit_plan( snapshot, run_directory, + maximum_audit_cost_usd="1", + reference_set_path=_reference_set_path(run_directory), workbook_sha256="a" * 64, snapshot_sha256=snapshot_sha256, reference_set_version="references-v1", - eligibility_identity="c" * 64, + eligibility_identity=REFERENCE_IDENTITY, pipeline=_fake_pipeline, repository=tmp_path, ) - plan_path = run_directory / "evaluation-manifest.json" - authorization = live_authorization_digest(plan["plan_digest"], "1") - with pytest.raises(AuditExecutionError, match="--live"): - execute_authorized_audit( - plan_path, + execute_audit_run( + run_directory, live=False, - authorization=authorization, api_key="secret", - max_audit_cost_usd="1", + approval_mode="interactive", + approved_at=APPROVED_AT, client=FakeClient([]), sleep=lambda _delay: None, ) - with pytest.raises(AuditExecutionError, match="authorization"): - execute_authorized_audit( - plan_path, + + with pytest.raises(AuditExecutionError, match="approval"): + execute_audit_run( + run_directory, live=True, - authorization="wrong", api_key="secret", - max_audit_cost_usd="1", + approval_mode="wrong", + approved_at=APPROVED_AT, client=FakeClient([]), sleep=lambda _delay: None, ) @@ -234,12 +283,12 @@ def test_live_gates_retries_and_format_failures( _success("not json"), ] ) - manifest = execute_authorized_audit( - plan_path, + manifest = execute_audit_run( + run_directory, live=True, - authorization=authorization, api_key="secret", - max_audit_cost_usd="1", + approval_mode="interactive", + approved_at=APPROVED_AT, client=client, sleep=lambda _delay: None, ) @@ -251,48 +300,110 @@ def test_live_gates_retries_and_format_failures( assert manifest["usage"]["input_tokens"] == 22 assert manifest["estimated_audit_cost_usd"] == "0.0000164" assert manifest["configuration"]["temperature"] is None + assert manifest["approval"] == { + "mode": "interactive", + "approved_at": APPROVED_AT, + } + assert "authorization_digest" not in manifest raw_attempts = json.loads( (run_directory / "raw-attempts.json").read_text(encoding="utf-8") ) assert len(raw_attempts["requests"][0]["attempts"]) == 2 - with pytest.raises(AuditExecutionError, match="already been used"): - execute_authorized_audit( - plan_path, + with pytest.raises(AuditExecutionError, match="fresh evaluation run"): + execute_audit_run( + run_directory, live=True, - authorization=authorization, api_key="secret", - max_audit_cost_usd="1", + approval_mode="interactive", + approved_at=APPROVED_AT, client=FakeClient([]), sleep=lambda _delay: None, ) +@pytest.mark.parametrize( + "changed_artifact", + ("plan", "snapshot", "reference", "prompt"), +) +def test_live_execution_revalidates_frozen_evidence_before_requests( + tmp_path: Path, + changed_artifact: str, +) -> None: + """Any planned evidence drift fails closed before a provider request.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "run" + reference_path = _reference_set_path(run_directory) + build_audit_plan( + snapshot, + run_directory, + maximum_audit_cost_usd="1", + reference_set_path=reference_path, + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity=REFERENCE_IDENTITY, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + if changed_artifact == "plan": + path = run_directory / "evaluation-manifest.json" + value = json.loads(path.read_text(encoding="utf-8")) + value["maximum_audit_cost_usd"] = "2" + path.write_text(json.dumps(value), encoding="utf-8") + elif changed_artifact == "snapshot": + snapshot.write_text("changed", encoding="utf-8") + elif changed_artifact == "reference": + reference_path.write_text("{}", encoding="utf-8") + else: + prompt_path = run_directory / "prompts" / "heading_structure.json" + value = json.loads(prompt_path.read_text(encoding="utf-8")) + value["prompt_text"] = "changed" + prompt_path.write_text(json.dumps(value), encoding="utf-8") + client = FakeClient([]) + + with pytest.raises(AuditExecutionError): + execute_audit_run( + run_directory, + live=True, + api_key="secret", + approval_mode="auto", + approved_at=APPROVED_AT, + client=client, + sleep=lambda _delay: None, + ) + + assert client.outcomes == [] + + def test_cost_guard_is_checked_before_each_retry(tmp_path: Path) -> None: """A billed failure can exhaust the limit before an automatic retry.""" snapshot = tmp_path / "source.html" snapshot.write_text("", encoding="utf-8") snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() run_directory = tmp_path / "run" + limit = "0.0000001" plan = build_audit_plan( snapshot, run_directory, + maximum_audit_cost_usd=limit, + reference_set_path=_reference_set_path(run_directory), workbook_sha256="a" * 64, snapshot_sha256=snapshot_sha256, reference_set_version="references-v1", - eligibility_identity="c" * 64, + eligibility_identity=REFERENCE_IDENTITY, pipeline=_fake_pipeline, repository=tmp_path, ) - limit = "0.0000001" client = FakeClient([_transient(), _success("[]")]) - manifest = execute_authorized_audit( - run_directory / "evaluation-manifest.json", + manifest = execute_audit_run( + run_directory, live=True, - authorization=live_authorization_digest(plan["plan_digest"], limit), api_key="secret", - max_audit_cost_usd=limit, + approval_mode="auto", + approved_at=APPROVED_AT, client=client, sleep=lambda _delay: None, ) @@ -323,21 +434,23 @@ def test_permanent_provider_failures_are_not_retried( plan = build_audit_plan( snapshot, run_directory, + maximum_audit_cost_usd="1", + reference_set_path=_reference_set_path(run_directory), workbook_sha256="a" * 64, snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), reference_set_version="references-v1", - eligibility_identity="c" * 64, + eligibility_identity=REFERENCE_IDENTITY, pipeline=_fake_pipeline, repository=tmp_path, ) client = FakeClient([outcome, _success("[]")]) - manifest = execute_authorized_audit( - run_directory / "evaluation-manifest.json", + manifest = execute_audit_run( + run_directory, live=True, - authorization=live_authorization_digest(plan["plan_digest"], "1"), api_key="secret", - max_audit_cost_usd="1", + approval_mode="auto", + approved_at=APPROVED_AT, client=client, sleep=lambda _delay: None, ) @@ -357,20 +470,22 @@ def test_exhausted_transient_failures_make_the_run_unranked( plan = build_audit_plan( snapshot, run_directory, + maximum_audit_cost_usd="1", + reference_set_path=_reference_set_path(run_directory), workbook_sha256="a" * 64, snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), reference_set_version="references-v1", - eligibility_identity="c" * 64, + eligibility_identity=REFERENCE_IDENTITY, pipeline=_fake_pipeline, repository=tmp_path, ) - manifest = execute_authorized_audit( - run_directory / "evaluation-manifest.json", + manifest = execute_audit_run( + run_directory, live=True, - authorization=live_authorization_digest(plan["plan_digest"], "1"), api_key="secret", - max_audit_cost_usd="1", + approval_mode="auto", + approved_at=APPROVED_AT, client=FakeClient([_transient(), _transient(), _transient()]), sleep=lambda _delay: None, ) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index aa4f4f6..03a2009 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,4 +1,5 @@ import hashlib +import io import json import os import subprocess @@ -15,8 +16,10 @@ import pytest from vision_aid.evaluation import audit as evaluation_audit -from vision_aid.evaluation.audit import live_authorization_digest from vision_aid.evaluation.cli import main as evaluation_main +from vision_aid.evaluation.schemas import ReferenceSet +from vision_aid.evaluation.serialization import save_reference_set +from vision_aid.evaluation.workspace import PrivateWorkspace PROJECT_ROOT = Path(__file__).parents[1] SNAPSHOT_BODY = ( @@ -106,6 +109,48 @@ def run_cli( ) +class TerminalInput(io.StringIO): + """Provide controlled input with an explicit terminal capability.""" + + def __init__(self, value: str, *, terminal: bool = True) -> None: + super().__init__(value) + self.terminal = terminal + + def isatty(self) -> bool: + """Report whether interactive confirmation is available.""" + return self.terminal + + +def create_planned_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> Path: + """Plan a minimal evaluation run through the public CLI boundary.""" + monkeypatch.chdir(tmp_path) + workspace = PrivateWorkspace.from_current_directory() + workspace.initialize() + references = ReferenceSet( + version="synthetic-v1", + workbook_filename="synthetic.xlsx", + workbook_sha256="a" * 64, + homepage_url="https://example.test/", + references=(), + ) + save_reference_set(workspace.root / "references" / "approved.json", references) + snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" + snapshot_directory.mkdir(parents=True) + snapshot = snapshot_directory / "source.html" + snapshot.write_text("synthetic", encoding="utf-8") + (snapshot_directory / "metadata.json").write_text( + json.dumps({"sha256": hashlib.sha256(snapshot.read_bytes()).hexdigest()}), + encoding="utf-8", + ) + assert evaluation_main(["audit", "--max-audit-cost-usd", "0.01"]) == 0 + capsys.readouterr() + return next((workspace.root / "runs").iterdir()) + + @pytest.fixture def synthetic_workbook(tmp_path: Path) -> Path: """Create the minimal authorized workbook placeholder used by CLI tests.""" @@ -459,6 +504,152 @@ def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( assert "No network requests were made" in result.stdout +@pytest.mark.parametrize("option", ("--plan", "--authorize", "--bundle")) +def test_obsolete_workflow_options_are_rejected(option: str) -> None: + """Removed plan, authorization, and report-bundle flags stay removed.""" + workflow = "report" if option == "--bundle" else "audit" + prefix = ("--run-dir", "obsolete-run") if workflow == "report" else () + result = run_cli(workflow, *prefix, option, "obsolete.json") + + assert result.returncode == 2 + assert "unrecognized arguments" in result.stderr + + +def test_interactive_live_approval_accepts_trimmed_case_insensitive_yes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A terminal operator can approve the displayed run by typing yes.""" + run_directory = create_planned_run(tmp_path, monkeypatch, capsys) + + class SyntheticClient: + """Return empty successful findings without provider access.""" + + def __init__(self, **_kwargs: object) -> None: + pass + + def call(self, _prompt: str) -> dict[str, object]: + return { + "success": True, + "response": "[]", + "usage": {}, + "duration_seconds": 0, + } + + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setattr(sys, "stdin", TerminalInput(" YeS \n")) + monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) + + assert ( + evaluation_main( + ["audit", "--live", "--run-dir", str(run_directory)] + ) + == 0 + ) + + output = capsys.readouterr().out + manifest = json.loads( + (run_directory / "live-manifest.json").read_text(encoding="utf-8") + ) + assert "Evaluation execution summary" in output + assert "Approval mode: interactive" in output + assert manifest["approval"]["mode"] == "interactive" + + +@pytest.mark.parametrize( + ("response", "terminal", "message"), + ( + ("no\n", True, "was not 'yes'"), + ("\n", True, "was not 'yes'"), + ("", True, "ended before confirmation"), + ("yes\n", False, "requires a terminal"), + ), +) +def test_failed_live_approval_stops_before_client_or_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + response: str, + terminal: bool, + message: str, +) -> None: + """Ambiguous, missing, or non-terminal input cannot begin a live run.""" + run_directory = create_planned_run(tmp_path, monkeypatch, capsys) + + class ForbiddenClient: + """Fail if approval rejection constructs a provider client.""" + + def __init__(self, **_kwargs: object) -> None: + raise AssertionError("provider client was constructed") + + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setattr( + sys, + "stdin", + TerminalInput(response, terminal=terminal), + ) + monkeypatch.setattr(evaluation_audit, "AuditRequestClient", ForbiddenClient) + + with pytest.raises(SystemExit) as raised: + evaluation_main( + ["audit", "--live", "--run-dir", str(run_directory)] + ) + + assert raised.value.code == 2 + captured = capsys.readouterr() + assert "Evaluation execution summary" in captured.out + assert message in captured.err + for name in ( + "raw-attempts.json", + "canonical-audit-findings.json", + "live-manifest.json", + ): + assert not (run_directory / name).exists() + + +def test_live_option_combinations_keep_every_spending_gate_mandatory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Automation cannot blur planning, live intent, credentials, or cost.""" + run_directory = create_planned_run(tmp_path, monkeypatch, capsys) + + with pytest.raises(SystemExit): + evaluation_main(["audit", "--auto-approve"]) + assert "only with --live" in capsys.readouterr().err + + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + with pytest.raises(SystemExit): + evaluation_main( + [ + "audit", + "--live", + "--run-dir", + str(run_directory), + "--auto-approve", + "--max-audit-cost-usd", + "1", + ] + ) + assert "reuses the saved cost guardrail" in capsys.readouterr().err + + monkeypatch.delenv("OPENAI_API_KEY") + with pytest.raises(SystemExit): + evaluation_main( + [ + "audit", + "--live", + "--run-dir", + str(run_directory), + "--auto-approve", + ] + ) + assert "requires OPENAI_API_KEY" in capsys.readouterr().err + assert not (run_directory / "live-manifest.json").exists() + + def test_cli_completes_the_synthetic_private_workflow( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -531,13 +722,33 @@ def test_cli_completes_the_synthetic_private_workflow( ) assert planned.returncode == 0, planned.stderr assert "DRY RUN" in planned.stdout - assert "Live authorization digest" in planned.stdout + assert "Evaluation execution summary" in planned.stdout + assert "Maximum audit cost: $0.01" in planned.stdout + assert "Evaluation run:" in planned.stdout approved_path = workspace / "references" / "approved.json" approved = json.loads(approved_path.read_text(encoding="utf-8")) - run_directory = workspace / "runs" / "dry-run" + run_directories = list((workspace / "runs").iterdir()) + assert len(run_directories) == 1 + run_directory = run_directories[0] plan_path = run_directory / "evaluation-manifest.json" run_manifest = json.loads(plan_path.read_text(encoding="utf-8")) + assert run_directory.name == run_manifest["run_id"] + assert run_manifest["maximum_audit_cost_usd"] == "0.01" + frozen_reference = run_directory / "reference-set.json" + assert frozen_reference.read_bytes() == approved_path.read_bytes() + first_plan_bytes = plan_path.read_bytes() + repeated_plan = run_cli( + "audit", + "--max-audit-cost-usd", + "0.01", + cwd=tmp_path, + ) + assert repeated_plan.returncode == 0, repeated_plan.stderr + repeated_directories = list((workspace / "runs").iterdir()) + assert len(repeated_directories) == 2 + assert len({path.name for path in repeated_directories}) == 2 + assert plan_path.read_bytes() == first_plan_bytes class SyntheticClient: """Return deterministic successful responses without network access.""" @@ -561,10 +772,6 @@ def call(self, _prompt: str) -> dict[str, object]: "stop_reason": "stop", } - maximum_cost = "0.01" - authorization = live_authorization_digest( - run_manifest["plan_digest"], maximum_cost - ) monkeypatch.chdir(tmp_path) monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) @@ -573,28 +780,67 @@ def call(self, _prompt: str) -> dict[str, object]: [ "audit", "--live", - "--plan", - str(plan_path), - "--authorize", - authorization, - "--max-audit-cost-usd", - maximum_cost, + "--run-dir", + str(run_directory), + "--auto-approve", ] ) == 0 ) - capsys.readouterr() + live_output = capsys.readouterr().out + for expected in ( + "Model: gpt-5.6-luna", + "Endpoint: chat.completions", + "Reasoning effort: medium", + "Benchmark snapshot SHA-256:", + "Reference set:", + "Prompts (", + "Estimated input tokens:", + "Retry policy:", + "Maximum audit cost: $0.01", + f"Destination: {run_directory}", + "Approval mode: auto", + ): + assert expected in live_output + live_manifest = json.loads( + (run_directory / "live-manifest.json").read_text(encoding="utf-8") + ) + assert live_manifest["approval"]["mode"] == "auto" + assert "approved_at" in live_manifest["approval"] + assert "authorization_digest" not in live_manifest + assert "authorized_plan_digest" not in live_manifest + live_artifacts = { + name: (run_directory / name).read_bytes() + for name in ( + "raw-attempts.json", + "canonical-audit-findings.json", + "live-manifest.json", + ) + } + with pytest.raises(SystemExit) as occupied: + evaluation_main( + [ + "audit", + "--live", + "--run-dir", + str(run_directory), + "--auto-approve", + ] + ) + assert occupied.value.code == 2 + assert "fresh evaluation run" in capsys.readouterr().err + for name, original_bytes in live_artifacts.items(): + assert (run_directory / name).read_bytes() == original_bytes finding_path = run_directory / "canonical-audit-findings.json" - matches_workbook = workspace / "reviews" / "matches.xlsx" + review_directory = workspace / "reviews" / run_manifest["run_id"] + matches_workbook = review_directory / "audit-matches.xlsx" generated = run_cli( "review", - "--reference-set", - str(approved_path), - "--findings", - str(finding_path), - "--matches-workbook", - str(matches_workbook), + "--run-dir", + str(run_directory), + "--kind", + "audit", cwd=tmp_path, ) assert generated.returncode == 0, generated.stderr @@ -617,44 +863,87 @@ def call(self, _prompt: str) -> dict[str, object]: match_review.save(matches_workbook) imported = run_cli( "review", - "--reference-set", - str(approved_path), - "--findings", - str(finding_path), - "--matches-workbook", - str(matches_workbook), + "--run-dir", + str(run_directory), + "--kind", + "audit", "--import-matches", cwd=tmp_path, ) assert imported.returncode == 0, imported.stderr - - empty_matches = run_directory / "empty-matches.json" - empty_matches.write_text("[]", encoding="utf-8") - bundle_path = workspace / "synthetic-bundle.json" - bundle_path.write_text( - json.dumps( - { - "reference_set": "references/approved.json", - "audit_findings": "runs/dry-run/canonical-audit-findings.json", - "programmatic_findings": "runs/dry-run/canonical-programmatic.json", - "audit_matches": "reviews/match-decisions.json", - "programmatic_matches": "runs/dry-run/empty-matches.json", - "run_manifest": "runs/dry-run/live-manifest.json", - } - ), + programmatic_generated = run_cli( + "review", + "--run-dir", + str(run_directory), + "--kind", + "programmatic", + cwd=tmp_path, + ) + assert programmatic_generated.returncode == 0, programmatic_generated.stderr + programmatic_workbook = review_directory / "programmatic-matches.xlsx" + programmatic_review = openpyxl.load_workbook(programmatic_workbook) + programmatic_matches = programmatic_review["Matches"] + programmatic_columns = { + cell.value: cell.column for cell in programmatic_matches[1] + } + for row in range(2, programmatic_matches.max_row + 1): + for name, value in { + "decision": "rejected", + "rationale": "The deterministic finding is not the same defect.", + "reviewer": "reviewer@example.test", + "confidence": 0.9, + "timestamp": "2026-09-19T12:35:00Z", + }.items(): + programmatic_matches.cell(row, programmatic_columns[name], value) + programmatic_review.save(programmatic_workbook) + programmatic_imported = run_cli( + "review", + "--run-dir", + str(run_directory), + "--kind", + "programmatic", + "--import-matches", + cwd=tmp_path, + ) + assert programmatic_imported.returncode == 0, programmatic_imported.stderr + assert (review_directory / "audit-match-decisions.json").is_file() + assert ( + review_directory / "programmatic-match-decisions.json" + ).is_file() + live_manifest["format_failures"] = ["synthetic-format"] + (run_directory / "live-manifest.json").write_text( + json.dumps(live_manifest), encoding="utf-8", ) - reported = run_cli("report", "--bundle", str(bundle_path), cwd=tmp_path) + reported = run_cli( + "report", "--run-dir", str(run_directory), cwd=tmp_path + ) assert reported.returncode == 0, reported.stderr - assert (workspace / "reports" / f"{run_manifest['run_id']}.json").is_file() + json_report = workspace / "reports" / f"{run_manifest['run_id']}.json" + assert json_report.is_file() + report_data = json.loads(json_report.read_text(encoding="utf-8")) + assert report_data["parse_failures"] == ["synthetic-format"] + + programmatic_decisions = ( + review_directory / "programmatic-match-decisions.json" + ) + decision_bytes = programmatic_decisions.read_bytes() + programmatic_decisions.unlink() + unresolved = run_cli( + "report", "--run-dir", str(run_directory), cwd=tmp_path + ) + assert unresolved.returncode == 2 + assert "--kind programmatic" in unresolved.stderr + assert "--import-matches" in unresolved.stderr + programmatic_decisions.write_bytes(decision_bytes) findings = json.loads(finding_path.read_text(encoding="utf-8")) findings[0]["run_id"] = "different-run" finding_path.write_text(json.dumps(findings), encoding="utf-8") rejected_finding = run_cli( - "report", "--bundle", str(bundle_path), cwd=tmp_path + "report", "--run-dir", str(run_directory), cwd=tmp_path ) assert rejected_finding.returncode == 2 assert "do not belong to the verified run" in rejected_finding.stderr @@ -662,9 +951,9 @@ def call(self, _prompt: str) -> dict[str, object]: findings[0]["run_id"] = run_manifest["run_id"] finding_path.write_text(json.dumps(findings), encoding="utf-8") approved["references"][0]["eligibility_review"]["rationale"] = "Changed" - approved_path.write_text(json.dumps(approved), encoding="utf-8") + frozen_reference.write_text(json.dumps(approved), encoding="utf-8") rejected_eligibility = run_cli( - "report", "--bundle", str(bundle_path), cwd=tmp_path + "report", "--run-dir", str(run_directory), cwd=tmp_path ) assert rejected_eligibility.returncode == 2 - assert "eligibility decisions" in rejected_eligibility.stderr + assert "reference-set" in rejected_eligibility.stderr diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index fa84751..f09129c 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -22,7 +22,12 @@ normalize_prompt_response, ) from vision_aid.evaluation.pricing import PriceSchedule -from vision_aid.evaluation.serialization import json_digest, write_json +from vision_aid.evaluation.serialization import ( + json_digest, + load_reference_set, + reference_set_identity, + write_json, +) MODEL = "gpt-5.6-luna" REQUEST_CONFIG = AuditRequestConfig( @@ -38,6 +43,21 @@ "backoff_seconds": list(BACKOFF_SECONDS), "retryable": list(RETRYABLE_FAILURES), } +PLAN_FILENAME = "evaluation-manifest.json" +LIVE_OUTPUT_FILENAMES = ( + "raw-attempts.json", + "canonical-audit-findings.json", + "live-manifest.json", +) +FROZEN_CONFIGURATION = { + "model": MODEL, + "reasoning_effort": "medium", + "endpoint": "chat.completions", + "temperature": None, + "max_output_tokens": 8192, + "include_summaries": False, + "execution": "sequential", +} class AuditExecutionError(ValueError): @@ -57,6 +77,36 @@ def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def _positive_cost(value: str) -> Decimal: + """Parse one strictly positive audit-cost guardrail.""" + try: + cost_limit = Decimal(value) + except Exception as error: + raise AuditExecutionError( + "The audit cost limit must be a decimal amount." + ) from error + if not cost_limit.is_finite() or cost_limit <= 0: + raise AuditExecutionError( + "The audit cost limit must be greater than zero." + ) + return cost_limit + + +def _recorded_path(path: Path, run_directory: Path) -> str: + """Prefer a run-relative evidence path while supporting external inputs.""" + resolved = path.resolve() + try: + return str(resolved.relative_to(run_directory.resolve())) + except ValueError: + return str(resolved) + + +def _resolve_recorded_path(value: str, run_directory: Path) -> Path: + """Resolve one plan artifact path against its evaluation run.""" + path = Path(value) + return path if path.is_absolute() else run_directory / path + + def _repository_state(repository: Path) -> dict[str, Any]: """Return the source-control commit and dirty state when available.""" try: @@ -83,6 +133,8 @@ def build_audit_plan( snapshot_path: Path, output_directory: Path, *, + maximum_audit_cost_usd: str, + reference_set_path: Path, workbook_sha256: str, snapshot_sha256: str, reference_set_version: str, @@ -91,8 +143,10 @@ def build_audit_plan( pipeline: Pipeline | None = None, repository: Path | None = None, schedule: PriceSchedule | None = None, + run_id: str | None = None, ) -> dict[str, Any]: - """Run the existing pipeline dry and freeze an authorization summary.""" + """Run the existing pipeline dry and freeze the billable intent.""" + cost_limit = _positive_cost(maximum_audit_cost_usd) if _sha256(snapshot_path) != snapshot_sha256: raise AuditExecutionError( "Benchmark snapshot checksum drift detected before audit planning." @@ -132,14 +186,29 @@ def build_audit_plan( price_schedule = schedule or PriceSchedule.default() parser_path = Path(__file__).with_name("normalization.py") now = datetime.now(UTC).isoformat().replace("+00:00", "Z") - run_id = "dry-" + json_digest([snapshot_sha256, now])[:16] + frozen_references = load_reference_set(reference_set_path) + if frozen_references.workbook_sha256 != workbook_sha256: + raise AuditExecutionError( + "Frozen reference-set workbook identity does not match the plan." + ) + if frozen_references.version != reference_set_version: + raise AuditExecutionError( + "Frozen reference-set version does not match the plan." + ) + if reference_set_identity(frozen_references) != eligibility_identity: + raise AuditExecutionError( + "Frozen eligibility decisions do not match the plan." + ) + resolved_run_id = run_id or ( + "run-" + json_digest([snapshot_sha256, now])[:16] + ) raw_programmatic_path = output_directory / "programmatic_findings.json" raw_programmatic = json.loads( raw_programmatic_path.read_text(encoding="utf-8") ) canonical_programmatic = normalize_programmatic_findings( raw_programmatic, - run_id=run_id, + run_id=resolved_run_id, page_url=homepage_url, ) canonical_programmatic_path = ( @@ -152,21 +221,13 @@ def build_audit_plan( plan: dict[str, Any] = { "kind": "model-evaluation-audit-plan", "created_at": now, - "run_id": run_id, + "run_id": resolved_run_id, "homepage_url": homepage_url, "workbook_sha256": workbook_sha256, "snapshot_sha256": snapshot_sha256, "reference_set_version": reference_set_version, "eligibility_identity": eligibility_identity, - "configuration": { - "model": MODEL, - "reasoning_effort": "medium", - "endpoint": "chat.completions", - "temperature": None, - "max_output_tokens": 8192, - "include_summaries": False, - "execution": "sequential", - }, + "configuration": FROZEN_CONFIGURATION, "filters": production_manifest.get("pass1_filters_active", []), "skipped_prompts": production_manifest.get("prompts_skipped", []), "request_count": len(prompts), @@ -179,6 +240,15 @@ def build_audit_plan( "pricing_version": price_schedule.version, "pricing_identity": price_schedule.identity, "retry_policy": RETRY_POLICY, + "maximum_audit_cost_usd": format(cost_limit.normalize(), "f"), + "snapshot_evidence": { + "path": _recorded_path(snapshot_path, output_directory), + "sha256": snapshot_sha256, + }, + "reference_evidence": { + "path": _recorded_path(reference_set_path, output_directory), + "sha256": _sha256(reference_set_path), + }, "repository": _repository_state(repository or Path.cwd()), "programmatic_findings": { "count": len(canonical_programmatic), @@ -205,16 +275,111 @@ def build_audit_plan( } ) plan["plan_digest"] = json_digest(plan) - write_json(output_directory / "evaluation-manifest.json", plan) + write_json(output_directory / PLAN_FILENAME, plan) return plan -def live_authorization_digest(plan_digest: str, maximum_cost_usd: str) -> str: - """Bind one explicit authorization to a plan and cost guardrail.""" - cost = format(Decimal(maximum_cost_usd).normalize(), "f") - return json_digest( - {"plan_digest": plan_digest, "maximum_audit_cost_usd": cost} +def load_verified_audit_plan( + run_directory: Path, + *, + schedule: PriceSchedule | None = None, +) -> dict[str, Any]: + """Load an evaluation run and revalidate every frozen plan identity.""" + run_directory = run_directory.resolve() + plan_path = run_directory / PLAN_FILENAME + plan = json.loads(plan_path.read_text(encoding="utf-8")) + recorded_digest = plan.get("plan_digest") + digest_input = dict(plan) + digest_input.pop("plan_digest", None) + if recorded_digest != json_digest(digest_input): + raise AuditExecutionError( + "The saved evaluation plan has changed since creation." + ) + if plan.get("configuration") != FROZEN_CONFIGURATION: + raise AuditExecutionError( + "The planned configuration is not the frozen Luna POC." + ) + _positive_cost(str(plan.get("maximum_audit_cost_usd", ""))) + price_schedule = schedule or PriceSchedule.default() + if plan.get("pricing_identity") != price_schedule.identity: + raise AuditExecutionError( + "The price schedule differs from the evaluation plan." + ) + parser_path = Path(__file__).with_name("normalization.py") + if plan.get("parser_hash") != _sha256(parser_path): + raise AuditExecutionError( + "The parser identity differs from the evaluation plan." + ) + snapshot = plan.get("snapshot_evidence") or {} + snapshot_path = _resolve_recorded_path( + str(snapshot.get("path", "")), run_directory ) + if ( + not snapshot_path.is_file() + or _sha256(snapshot_path) != snapshot.get("sha256") + or snapshot.get("sha256") != plan.get("snapshot_sha256") + ): + raise AuditExecutionError( + "Benchmark snapshot identity differs from the evaluation plan." + ) + reference = plan.get("reference_evidence") or {} + reference_path = _resolve_recorded_path( + str(reference.get("path", "")), run_directory + ) + if ( + not reference_path.is_file() + or _sha256(reference_path) != reference.get("sha256") + ): + raise AuditExecutionError( + "Frozen reference-set bytes differ from the evaluation plan." + ) + reference_set = load_reference_set(reference_path) + if ( + reference_set.workbook_sha256 != plan.get("workbook_sha256") + or reference_set.version != plan.get("reference_set_version") + or reference_set_identity(reference_set) + != plan.get("eligibility_identity") + ): + raise AuditExecutionError( + "Frozen reference-set identities differ from the evaluation plan." + ) + for prompt in plan.get("prompts", ()): + name = str(prompt["name"]) + prompt_path = run_directory / str(prompt["path"]) + prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) + prompt_text = str(prompt_data["prompt_text"]) + if ( + hashlib.sha256(prompt_text.encode()).hexdigest() + != plan["prompt_hashes"].get(name) + ): + raise AuditExecutionError( + f"Prompt content drift detected for {name}." + ) + programmatic = plan.get("programmatic_findings") or {} + programmatic_path = run_directory / str(programmatic.get("path", "")) + if ( + not programmatic_path.is_file() + or _sha256(programmatic_path) != programmatic.get("sha256") + ): + raise AuditExecutionError( + "Programmatic findings differ from the evaluation plan." + ) + return plan + + +def ensure_live_destination_available(run_directory: Path) -> None: + """Refuse to overwrite any evidence from an earlier live execution.""" + occupied = [ + run_directory / name + for name in LIVE_OUTPUT_FILENAMES + if (run_directory / name).exists() + ] + if occupied: + names = ", ".join(path.name for path in occupied) + raise AuditExecutionError( + f"Live output already exists ({names}). Create a fresh " + "evaluation run before executing again." + ) def _failure_classification(result: dict[str, Any]) -> str: @@ -261,89 +426,39 @@ def _add_usage(total: dict[str, int], result: dict[str, Any]) -> None: total[key] += int(usage.get(key, 0) or 0) -def execute_authorized_audit( - plan_path: Path, +def execute_audit_run( + run_directory: Path, *, live: bool = False, - authorization: str | None, api_key: str | None, - max_audit_cost_usd: str | None, + approval_mode: str, + approved_at: str, client: RequestClient | None = None, schedule: PriceSchedule | None = None, sleep: Callable[[float], None] = time.sleep, ) -> dict[str, Any]: - """Execute an exact reviewed plan after every spending gate passes.""" + """Execute one verified evaluation run after point-of-use approval.""" if not live: raise AuditExecutionError( "Live execution requires the explicit --live flag." ) if not api_key: raise AuditExecutionError("Live execution requires OPENAI_API_KEY.") - if max_audit_cost_usd is None: + if approval_mode not in {"interactive", "auto"}: raise AuditExecutionError( - "Live execution requires --max-audit-cost-usd as a spending " - "guardrail." - ) - try: - cost_limit = Decimal(max_audit_cost_usd) - except Exception as error: - raise AuditExecutionError( - "The audit cost limit must be a decimal amount." - ) from error - if cost_limit <= 0: - raise AuditExecutionError( - "The audit cost limit must be greater than zero." - ) - plan = json.loads(plan_path.read_text(encoding="utf-8")) - recorded_digest = plan.get("plan_digest") - digest_input = dict(plan) - digest_input.pop("plan_digest", None) - if recorded_digest != json_digest(digest_input): - raise AuditExecutionError( - "The saved dry-run summary has changed since creation." - ) - expected_authorization = live_authorization_digest( - str(recorded_digest), - str(cost_limit), - ) - if authorization != expected_authorization: - raise AuditExecutionError( - "Live authorization does not match the summarized configuration " - "and cost guardrail." - ) - if plan.get("configuration") != { - "model": MODEL, - "reasoning_effort": "medium", - "endpoint": "chat.completions", - "temperature": None, - "max_output_tokens": 8192, - "include_summaries": False, - "execution": "sequential", - }: - raise AuditExecutionError( - "The dry-run configuration is not the frozen Luna POC." + "Live execution requires interactive or automatic approval." ) + run_directory = run_directory.resolve() + ensure_live_destination_available(run_directory) + plan = load_verified_audit_plan(run_directory, schedule=schedule) + recorded_digest = plan["plan_digest"] + cost_limit = _positive_cost(plan["maximum_audit_cost_usd"]) price_schedule = schedule or PriceSchedule.default() - if plan.get("pricing_identity") != price_schedule.identity: - raise AuditExecutionError( - "The price schedule differs from the authorized plan." - ) request_client = client or AuditRequestClient( api_key=api_key, request_config=REQUEST_CONFIG, ) - output_directory = plan_path.parent - authorization_receipt = output_directory / ( - f".authorization-{expected_authorization}.used" - ) - try: - authorization_receipt.touch(exist_ok=False) - except FileExistsError as error: - raise AuditExecutionError( - "This live authorization has already been used; a rerun requires " - "a " - "new dry-run summary and authorization." - ) from error + output_directory = run_directory requests: list[dict[str, Any]] = [] totals = _empty_usage() format_failures: list[str] = [] @@ -439,9 +554,10 @@ def execute_authorized_audit( manifest = { **plan, "kind": "model-evaluation-live-run", - "authorized_plan_digest": recorded_digest, - "authorization_digest": expected_authorization, - "maximum_audit_cost_usd": str(cost_limit), + "approval": { + "mode": approval_mode, + "approved_at": approved_at, + }, "completed_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), "complete": complete, "rankable": complete, diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index f40de60..c1c21bc 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -3,7 +3,9 @@ import argparse import json import os +import sys from collections.abc import Callable, Sequence +from datetime import UTC, datetime from pathlib import Path from vision_aid.evaluation.matching import ( @@ -104,19 +106,15 @@ def build_parser() -> argparse.ArgumentParser: "--live", action="store_true", help="enable billable execution" ) audit.add_argument( - "--plan", type=Path, help="exact dry-run manifest to execute" - ) - audit.add_argument( - "--authorize", help="authorization digest shown by dry run" - ) - audit.add_argument( - "--max-audit-cost-usd", help="required live cost guardrail" + "--auto-approve", + action="store_true", + help="deliberately approve live execution without a terminal prompt", ) audit.add_argument( - "--reference-set", type=Path, help="approved reference-set JSON" + "--max-audit-cost-usd", help="required planning cost guardrail" ) audit.add_argument( - "--run-dir", type=Path, help="private output directory for the plan" + "--run-dir", type=Path, help="planned evaluation run to execute" ) audit.set_defaults(handler=_audit) @@ -135,6 +133,14 @@ def build_parser() -> argparse.ArgumentParser: review.add_argument( "--findings", type=Path, help="canonical findings JSON" ) + review.add_argument( + "--run-dir", type=Path, help="evaluation run being reviewed" + ) + review.add_argument( + "--kind", + choices=("audit", "programmatic"), + help="finding collection being reviewed", + ) review.add_argument( "--matches-workbook", type=Path, @@ -147,6 +153,11 @@ def build_parser() -> argparse.ArgumentParser: "import decisions from --matches-workbook instead of generating it" ), ) + review.add_argument( + "--match-decisions", + type=Path, + help="validated match-decision JSON output override", + ) review.set_defaults(handler=_review) report = workflows.add_parser( @@ -154,9 +165,7 @@ def build_parser() -> argparse.ArgumentParser: help="generate private evaluation reports", ) report.add_argument( - "--bundle", - type=Path, - help="private JSON bundle containing score inputs", + "--run-dir", type=Path, required=True, help="evaluation run to report" ) report.add_argument( "--output-dir", type=Path, help="private report directory" @@ -207,30 +216,118 @@ def _prepare(arguments: argparse.Namespace) -> int: return 0 +def _render_execution_summary( + plan: dict[str, object], + run_directory: Path, + approval_mode: str, +) -> str: + """Render the complete billable intent for planning or live approval.""" + configuration = plan["configuration"] + assert isinstance(configuration, dict) + prompts = plan["prompts"] + assert isinstance(prompts, list) + prompt_names = ", ".join(str(item["name"]) for item in prompts) + retry_policy = plan["retry_policy"] + assert isinstance(retry_policy, dict) + return "\n".join( + ( + "Evaluation execution summary", + f" Model: {configuration['model']}", + f" Endpoint: {configuration['endpoint']}", + f" Reasoning effort: {configuration['reasoning_effort']}", + f" Benchmark snapshot SHA-256: {plan['snapshot_sha256']}", + f" Reference set: {plan['reference_set_version']}", + f" Reference identity: {plan['eligibility_identity']}", + f" Prompts ({plan['request_count']}): {prompt_names}", + f" Estimated input tokens: {plan['estimated_input_tokens']}", + " Retry policy: " + f"{retry_policy['maximum_additional_attempts']} additional " + f"attempts; backoff {retry_policy['backoff_seconds']}; " + f"retryable {retry_policy['retryable']}", + f" Maximum audit cost: ${plan['maximum_audit_cost_usd']}", + f" Destination: {run_directory.resolve()}", + f" Approval mode: {approval_mode}", + ) + ) + + +def _approval_timestamp() -> str: + """Return the UTC timestamp recorded for live-run approval.""" + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _confirm_live_execution() -> str: + """Require exact, terminal-originated point-of-use confirmation.""" + if not sys.stdin.isatty(): + raise PrivateInputError( + "Interactive live-run approval requires a terminal. Use " + "--auto-approve only for deliberate automation." + ) + try: + response = input("Type 'yes' to begin billable execution: ") + except EOFError as error: + raise PrivateInputError( + "Live-run approval ended before confirmation; no request was made." + ) from error + if response.strip().casefold() != "yes": + raise PrivateInputError( + "Live-run approval was not 'yes'; no request was made." + ) + return _approval_timestamp() + + def _audit(arguments: argparse.Namespace) -> int: - """Build a no-cost plan or execute its exact authorized configuration.""" + """Plan a no-cost evaluation run or execute one after explicit approval.""" + from vision_aid.evaluation.audit import ( + build_audit_plan, + ensure_live_destination_available, + execute_audit_run, + load_verified_audit_plan, + ) + workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() + if arguments.auto_approve and not arguments.live: + raise PrivateInputError("--auto-approve is accepted only with --live.") if arguments.live: - from vision_aid.evaluation.audit import execute_authorized_audit - - if arguments.plan is None: + if arguments.run_dir is None: + raise PrivateInputError("--live requires --run-dir from planning.") + if arguments.max_audit_cost_usd is not None: + raise PrivateInputError( + "Live execution reuses the saved cost guardrail; do not pass " + "--max-audit-cost-usd." + ) + run_directory = arguments.run_dir.resolve() + plan = load_verified_audit_plan(run_directory) + ensure_live_destination_available(run_directory) + if not os.environ.get("OPENAI_API_KEY"): raise PrivateInputError( - "--live requires --plan from a reviewed dry run." + "Live execution requires OPENAI_API_KEY in addition to " + "approval." ) - manifest = execute_authorized_audit( - arguments.plan, + approval_mode = "auto" if arguments.auto_approve else "interactive" + print( + _render_execution_summary(plan, run_directory, approval_mode), + flush=True, + ) + approved_at = ( + _approval_timestamp() + if arguments.auto_approve + else _confirm_live_execution() + ) + manifest = execute_audit_run( + run_directory, live=True, - authorization=arguments.authorize, api_key=os.environ.get("OPENAI_API_KEY"), - max_audit_cost_usd=arguments.max_audit_cost_usd, + approval_mode=approval_mode, + approved_at=approved_at, ) print(f"Live run complete: {manifest['complete']}") print(f"Estimated audit cost: ${manifest['estimated_audit_cost_usd']}") return 0 - reference_path = arguments.reference_set or ( - workspace.root / "references" / "approved.json" - ) + if arguments.run_dir is not None: + raise PrivateInputError("--run-dir is accepted only with --live.") + reference_path = workspace.root / "references" / "approved.json" snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" snapshot_path = snapshot_directory / "source.html" metadata_path = snapshot_directory / "metadata.json" @@ -239,11 +336,6 @@ def _audit(arguments: argparse.Namespace) -> int: and snapshot_path.is_file() and metadata_path.is_file() ): - from vision_aid.evaluation.audit import ( - build_audit_plan, - live_authorization_digest, - ) - reference_set = load_reference_set(reference_path) if any( item.eligibility_state != "accepted" @@ -255,31 +347,30 @@ def _audit(arguments: argparse.Namespace) -> int: snapshot_metadata = json.loads( metadata_path.read_text(encoding="utf-8") ) + if arguments.max_audit_cost_usd is None: + raise PrivateInputError( + "Audit planning requires --max-audit-cost-usd as a positive " + "spending guardrail." + ) eligibility_identity = reference_set_identity(reference_set) - run_directory = ( - arguments.run_dir or workspace.root / "runs" / "dry-run" - ) + run_id, run_directory = workspace.create_evaluation_run() + frozen_reference_path = run_directory / "reference-set.json" + save_reference_set(frozen_reference_path, reference_set) plan = build_audit_plan( snapshot_path, run_directory, + maximum_audit_cost_usd=arguments.max_audit_cost_usd, + reference_set_path=frozen_reference_path, workbook_sha256=reference_set.workbook_sha256, snapshot_sha256=snapshot_metadata["sha256"], reference_set_version=reference_set.version, eligibility_identity=eligibility_identity, homepage_url=reference_set.homepage_url, + run_id=run_id, ) print(f"DRY RUN: {plan['request_count']} sequential requests planned.") - print(f"Estimated input tokens: {plan['estimated_input_tokens']}") - print(f"Plan digest: {plan['plan_digest']}") - if arguments.max_audit_cost_usd: - authorization = live_authorization_digest( - plan["plan_digest"], - arguments.max_audit_cost_usd, - ) - print(f"Live authorization digest: {authorization}") - print( - f"Plan: {(run_directory / 'evaluation-manifest.json').resolve()}" - ) + print(_render_execution_summary(plan, run_directory, "pending")) + print(f"Evaluation run: {run_directory.resolve()}") return 0 print( "DRY RUN: No network requests were made and no API usage was incurred." @@ -303,27 +394,58 @@ def _review(arguments: argparse.Namespace) -> int: save_reference_set(approved_path, reviewed) print(f"Validated reference set: {approved_path.resolve()}") return 0 - if arguments.matches_workbook: - if not arguments.reference_set or not arguments.findings: + match_workflow_requested = any( + ( + arguments.run_dir, + arguments.kind, + arguments.reference_set, + arguments.findings, + arguments.matches_workbook, + arguments.match_decisions, + arguments.import_matches, + ) + ) + if match_workflow_requested: + if arguments.run_dir is None or arguments.kind is None: raise PrivateInputError( - "Match review requires --reference-set and --findings." + "Match review requires --run-dir and --kind " + "(audit or programmatic)." ) - references = load_reference_set(arguments.reference_set).references - findings = load_findings(arguments.findings) + from vision_aid.evaluation.audit import load_verified_audit_plan + + run_directory = arguments.run_dir.resolve() + plan = load_verified_audit_plan(run_directory) + run_id = str(plan["run_id"]) + review_directory = workspace.root / "reviews" / run_id + reference_path = arguments.reference_set or ( + run_directory / "reference-set.json" + ) + findings_path = arguments.findings or ( + run_directory + / ( + "canonical-audit-findings.json" + if arguments.kind == "audit" + else "canonical-programmatic.json" + ) + ) + workbook_path = arguments.matches_workbook or ( + review_directory / f"{arguments.kind}-matches.xlsx" + ) + decision_path = arguments.match_decisions or ( + review_directory / f"{arguments.kind}-match-decisions.json" + ) + references = load_reference_set(reference_path).references + findings = load_findings(findings_path) if arguments.import_matches: - decisions = HumanMatchReviewer(arguments.matches_workbook).review( + decisions = HumanMatchReviewer(workbook_path).review( references, findings, ) - output = workspace.root / "reviews" / "match-decisions.json" - save_match_decisions(output, decisions) - print(f"Validated match decisions: {output.resolve()}") + save_match_decisions(decision_path, decisions) + print(f"Validated match decisions: {decision_path.resolve()}") else: - export_match_workbook( - references, findings, arguments.matches_workbook - ) - resolved_review = arguments.matches_workbook.resolve() - print(f"Matches review workbook: {resolved_review}") + export_match_workbook(references, findings, workbook_path) + print(f"Matches review workbook: {workbook_path.resolve()}") return 0 print( f"Private review workspace: {(workspace.root / 'reviews').resolve()}" @@ -332,39 +454,68 @@ def _review(arguments: argparse.Namespace) -> int: def _report(arguments: argparse.Namespace) -> int: - """Expose the private report workflow shell.""" + """Discover verified run evidence and write every report projection.""" workspace = PrivateWorkspace.from_current_directory() workspace.require_initialized() - if arguments.bundle: - bundle = json.loads(arguments.bundle.read_text(encoding="utf-8")) - base = arguments.bundle.parent - reference_set = load_reference_set(base / bundle["reference_set"]) - if "run_manifest" not in bundle: + from vision_aid.evaluation.audit import load_verified_audit_plan + + run_directory = arguments.run_dir.resolve() + plan = load_verified_audit_plan(run_directory) + run_id = str(plan["run_id"]) + review_directory = workspace.root / "reviews" / run_id + inputs = { + "frozen reference set": run_directory / "reference-set.json", + "audit findings": run_directory / "canonical-audit-findings.json", + "programmatic findings": run_directory / "canonical-programmatic.json", + "audit match decisions": ( + review_directory / "audit-match-decisions.json" + ), + "programmatic match decisions": ( + review_directory / "programmatic-match-decisions.json" + ), + "live manifest": run_directory / "live-manifest.json", + } + for label, path in inputs.items(): + if path.is_file(): + continue + if "match decisions" in label: + kind = label.split()[0] raise PrivateInputError( - "Report bundles require run_manifest so benchmark identities " - "can be verified." + f"Missing {label} at {path}. Run " + f"'visionaid-evaluate review --run-dir {run_directory} " + f"--kind {kind}', complete the workbook, then rerun with " + "--import-matches." ) - score = score_evaluation( - reference_set, - load_findings(base / bundle["audit_findings"]), - load_findings(base / bundle["programmatic_findings"]), - load_match_decisions(base / bundle["audit_matches"]), - load_match_decisions(base / bundle["programmatic_matches"]), - load_verified_run_metadata( - base / bundle["run_manifest"], - reference_set, - ), - parse_failures=tuple(bundle.get("parse_failures", ())), + raise PrivateInputError( + f"Missing {label} at {path}; complete the evaluation run before " + "reporting." ) - output_directory = arguments.output_dir or workspace.root / "reports" - paths = write_reports(score, output_directory) - print(f"JSON report: {paths.json.resolve()}") - print(f"CSV report: {paths.csv.resolve()}") - print(f"Markdown report: {paths.markdown.resolve()}") - return 0 - print( - f"Private report workspace: {(workspace.root / 'reports').resolve()}" + reference_set = load_reference_set(inputs["frozen reference set"]) + live_manifest = json.loads( + inputs["live manifest"].read_text(encoding="utf-8") + ) + if ( + live_manifest.get("kind") != "model-evaluation-live-run" + or live_manifest.get("run_id") != run_id + or live_manifest.get("plan_digest") != plan.get("plan_digest") + ): + raise PrivateInputError( + "The live manifest does not belong to the verified evaluation run." + ) + score = score_evaluation( + reference_set, + load_findings(inputs["audit findings"]), + load_findings(inputs["programmatic findings"]), + load_match_decisions(inputs["audit match decisions"]), + load_match_decisions(inputs["programmatic match decisions"]), + load_verified_run_metadata(inputs["live manifest"], reference_set), + parse_failures=tuple(live_manifest.get("format_failures", ())), ) + output_directory = arguments.output_dir or workspace.root / "reports" + paths = write_reports(score, output_directory) + print(f"JSON report: {paths.json.resolve()}") + print(f"CSV report: {paths.csv.resolve()}") + print(f"Markdown report: {paths.markdown.resolve()}") return 0 diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py index 2e01e66..3fd0898 100644 --- a/vision_aid/evaluation/workspace.py +++ b/vision_aid/evaluation/workspace.py @@ -1,7 +1,9 @@ """Private filesystem conventions for model evaluation artifacts.""" from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path +from uuid import uuid4 DEFAULT_WORKBOOK_FILENAME = "Pristine Accessibility Defect Report.xlsx" DEFAULT_WORKSPACE = Path(".model-evaluation") @@ -51,6 +53,15 @@ def require_initialized(self) -> None: "with an authorized local workbook first." ) + def create_evaluation_run(self) -> tuple[str, Path]: + """Create and return one uniquely identified evaluation run.""" + self.require_initialized() + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") + run_id = f"run-{timestamp}-{uuid4().hex[:12]}" + run_directory = self.root / "runs" / run_id + run_directory.mkdir(exist_ok=False) + return run_id, run_directory + def find_worktree_root(start: Path) -> Path | None: """Find the containing Git worktree without invoking Git.""" From 6a1e6c0d1c8672250477632a1ddcb80c5b3af10e Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 18:37:41 -0400 Subject: [PATCH 021/126] fix(evaluation): bind execution to planned run destination --- tests/test_evaluation_audit.py | 72 +++++++++++++++ tests/test_evaluation_cli.py | 14 +++ vision_aid/evaluation/audit.py | 136 +++++++++++++++++------------ vision_aid/evaluation/cli.py | 86 ++++++++++++------ vision_aid/evaluation/workspace.py | 13 +++ 5 files changed, 235 insertions(+), 86 deletions(-) diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py index 24a7fa4..3128874 100644 --- a/tests/test_evaluation_audit.py +++ b/tests/test_evaluation_audit.py @@ -1,5 +1,6 @@ import hashlib import json +import shutil from pathlib import Path import pytest @@ -8,6 +9,8 @@ AuditExecutionError, build_audit_plan, execute_audit_run, + load_audit_plan, + load_verified_audit_plan, ) from vision_aid.evaluation.pricing import PriceSchedule from vision_aid.evaluation.schemas import ReferenceSet @@ -377,6 +380,75 @@ def test_live_execution_revalidates_frozen_evidence_before_requests( assert client.outcomes == [] +def test_copied_run_cannot_be_executed_as_a_fresh_destination( + tmp_path: Path, +) -> None: + """Copying an unexecuted run cannot create another billable run.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "workspace" / "runs" / "run" + build_audit_plan( + snapshot, + run_directory, + maximum_audit_cost_usd="1", + reference_set_path=_reference_set_path(run_directory), + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity=REFERENCE_IDENTITY, + pipeline=_fake_pipeline, + repository=tmp_path, + ) + copied_run = tmp_path / "other-workspace" / "runs" / "run" + shutil.copytree(run_directory, copied_run) + client = FakeClient([]) + + with pytest.raises(AuditExecutionError, match="live destination"): + execute_audit_run( + copied_run, + live=True, + api_key="secret", + approval_mode="auto", + approved_at=APPROVED_AT, + client=client, + sleep=lambda _delay: None, + ) + + assert client.outcomes == [] + + +def test_historical_plan_loading_does_not_require_current_execution_inputs( + tmp_path: Path, +) -> None: + """Review and reporting can use a run after pricing or snapshot changes.""" + snapshot = tmp_path / "source.html" + snapshot.write_text("", encoding="utf-8") + run_directory = tmp_path / "run" + current_schedule = PriceSchedule.default() + historical_schedule = PriceSchedule( + data=current_schedule.data, + identity="historical-pricing-identity", + ) + build_audit_plan( + snapshot, + run_directory, + maximum_audit_cost_usd="1", + reference_set_path=_reference_set_path(run_directory), + workbook_sha256="a" * 64, + snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), + reference_set_version="references-v1", + eligibility_identity=REFERENCE_IDENTITY, + pipeline=_fake_pipeline, + repository=tmp_path, + schedule=historical_schedule, + ) + snapshot.unlink() + + assert load_audit_plan(run_directory)["run_id"] == "run" + with pytest.raises(AuditExecutionError): + load_verified_audit_plan(run_directory) + + def test_cost_guard_is_checked_before_each_retry(tmp_path: Path) -> None: """A billed failure can exhaust the limit before an automatic retry.""" snapshot = tmp_path / "source.html" diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 03a2009..dcfccf0 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -615,6 +615,15 @@ def test_live_option_combinations_keep_every_spending_gate_mandatory( ) -> None: """Automation cannot blur planning, live intent, credentials, or cost.""" run_directory = create_planned_run(tmp_path, monkeypatch, capsys) + workspace = PrivateWorkspace.from_evaluation_run(run_directory) + existing_runs = {path.name for path in (workspace.root / "runs").iterdir()} + + with pytest.raises(SystemExit): + evaluation_main(["audit", "--max-audit-cost-usd", "0"]) + assert "greater than zero" in capsys.readouterr().err + assert { + path.name for path in (workspace.root / "runs").iterdir() + } == existing_runs with pytest.raises(SystemExit): evaluation_main(["audit", "--auto-approve"]) @@ -832,6 +841,11 @@ def call(self, _prompt: str) -> dict[str, object]: for name, original_bytes in live_artifacts.items(): assert (run_directory / name).read_bytes() == original_bytes + snapshot_path = ( + workspace / "snapshots" / "pristine-homepage" / "source.html" + ) + snapshot_path.unlink() + finding_path = run_directory / "canonical-audit-findings.json" review_directory = workspace / "reviews" / run_manifest["run_id"] matches_workbook = review_directory / "audit-matches.xlsx" diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index f09129c..c4ead6d 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -92,6 +92,11 @@ def _positive_cost(value: str) -> Decimal: return cost_limit +def normalize_audit_cost(value: str) -> str: + """Validate and normalize one planned maximum audit cost.""" + return format(_positive_cost(value).normalize(), "f") + + def _recorded_path(path: Path, run_directory: Path) -> str: """Prefer a run-relative evidence path while supporting external inputs.""" resolved = path.resolve() @@ -146,7 +151,7 @@ def build_audit_plan( run_id: str | None = None, ) -> dict[str, Any]: """Run the existing pipeline dry and freeze the billable intent.""" - cost_limit = _positive_cost(maximum_audit_cost_usd) + normalized_cost = normalize_audit_cost(maximum_audit_cost_usd) if _sha256(snapshot_path) != snapshot_sha256: raise AuditExecutionError( "Benchmark snapshot checksum drift detected before audit planning." @@ -199,9 +204,7 @@ def build_audit_plan( raise AuditExecutionError( "Frozen eligibility decisions do not match the plan." ) - resolved_run_id = run_id or ( - "run-" + json_digest([snapshot_sha256, now])[:16] - ) + resolved_run_id = run_id or output_directory.resolve().name raw_programmatic_path = output_directory / "programmatic_findings.json" raw_programmatic = json.loads( raw_programmatic_path.read_text(encoding="utf-8") @@ -240,7 +243,8 @@ def build_audit_plan( "pricing_version": price_schedule.version, "pricing_identity": price_schedule.identity, "retry_policy": RETRY_POLICY, - "maximum_audit_cost_usd": format(cost_limit.normalize(), "f"), + "maximum_audit_cost_usd": normalized_cost, + "execution_destination": str(output_directory.resolve()), "snapshot_evidence": { "path": _recorded_path(snapshot_path, output_directory), "sha256": snapshot_sha256, @@ -279,12 +283,8 @@ def build_audit_plan( return plan -def load_verified_audit_plan( - run_directory: Path, - *, - schedule: PriceSchedule | None = None, -) -> dict[str, Any]: - """Load an evaluation run and revalidate every frozen plan identity.""" +def load_audit_plan(run_directory: Path) -> dict[str, Any]: + """Load one run's immutable plan and self-contained evidence.""" run_directory = run_directory.resolve() plan_path = run_directory / PLAN_FILENAME plan = json.loads(plan_path.read_text(encoding="utf-8")) @@ -295,33 +295,14 @@ def load_verified_audit_plan( raise AuditExecutionError( "The saved evaluation plan has changed since creation." ) - if plan.get("configuration") != FROZEN_CONFIGURATION: + if plan.get("kind") != "model-evaluation-audit-plan": + raise AuditExecutionError("The evaluation run has an invalid plan kind.") + if plan.get("run_id") != run_directory.name: raise AuditExecutionError( - "The planned configuration is not the frozen Luna POC." + "The evaluation run identifier does not match its directory. " + "Create a fresh evaluation run instead of copying or renaming one." ) _positive_cost(str(plan.get("maximum_audit_cost_usd", ""))) - price_schedule = schedule or PriceSchedule.default() - if plan.get("pricing_identity") != price_schedule.identity: - raise AuditExecutionError( - "The price schedule differs from the evaluation plan." - ) - parser_path = Path(__file__).with_name("normalization.py") - if plan.get("parser_hash") != _sha256(parser_path): - raise AuditExecutionError( - "The parser identity differs from the evaluation plan." - ) - snapshot = plan.get("snapshot_evidence") or {} - snapshot_path = _resolve_recorded_path( - str(snapshot.get("path", "")), run_directory - ) - if ( - not snapshot_path.is_file() - or _sha256(snapshot_path) != snapshot.get("sha256") - or snapshot.get("sha256") != plan.get("snapshot_sha256") - ): - raise AuditExecutionError( - "Benchmark snapshot identity differs from the evaluation plan." - ) reference = plan.get("reference_evidence") or {} reference_path = _resolve_recorded_path( str(reference.get("path", "")), run_directory @@ -343,18 +324,6 @@ def load_verified_audit_plan( raise AuditExecutionError( "Frozen reference-set identities differ from the evaluation plan." ) - for prompt in plan.get("prompts", ()): - name = str(prompt["name"]) - prompt_path = run_directory / str(prompt["path"]) - prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) - prompt_text = str(prompt_data["prompt_text"]) - if ( - hashlib.sha256(prompt_text.encode()).hexdigest() - != plan["prompt_hashes"].get(name) - ): - raise AuditExecutionError( - f"Prompt content drift detected for {name}." - ) programmatic = plan.get("programmatic_findings") or {} programmatic_path = run_directory / str(programmatic.get("path", "")) if ( @@ -367,6 +336,68 @@ def load_verified_audit_plan( return plan +def _verified_prompt_text( + run_directory: Path, + plan: dict[str, Any], + prompt: dict[str, Any], +) -> str: + """Load one prompt only when its saved content identity still matches.""" + name = str(prompt["name"]) + prompt_path = run_directory / str(prompt["path"]) + prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) + prompt_text = str(prompt_data["prompt_text"]) + if ( + hashlib.sha256(prompt_text.encode()).hexdigest() + != plan["prompt_hashes"].get(name) + ): + raise AuditExecutionError(f"Prompt content drift detected for {name}.") + return prompt_text + + +def load_verified_audit_plan( + run_directory: Path, + *, + schedule: PriceSchedule | None = None, +) -> dict[str, Any]: + """Revalidate every point-of-use identity before live execution.""" + run_directory = run_directory.resolve() + plan = load_audit_plan(run_directory) + if plan.get("execution_destination") != str(run_directory): + raise AuditExecutionError( + "The live destination differs from the planned evaluation run. " + "Create a fresh evaluation run instead of copying one." + ) + if plan.get("configuration") != FROZEN_CONFIGURATION: + raise AuditExecutionError( + "The planned configuration is not the frozen Luna POC." + ) + price_schedule = schedule or PriceSchedule.default() + if plan.get("pricing_identity") != price_schedule.identity: + raise AuditExecutionError( + "The price schedule differs from the evaluation plan." + ) + parser_path = Path(__file__).with_name("normalization.py") + if plan.get("parser_hash") != _sha256(parser_path): + raise AuditExecutionError( + "The parser identity differs from the evaluation plan." + ) + snapshot = plan.get("snapshot_evidence") or {} + snapshot_path = _resolve_recorded_path( + str(snapshot.get("path", "")), run_directory + ) + if ( + not snapshot_path.is_file() + or _sha256(snapshot_path) != snapshot.get("sha256") + or snapshot.get("sha256") != plan.get("snapshot_sha256") + ): + raise AuditExecutionError( + "Benchmark snapshot identity differs from the evaluation plan." + ) + for prompt in plan.get("prompts", ()): + _verified_prompt_text(run_directory, plan, prompt) + return plan + + def ensure_live_destination_available(run_directory: Path) -> None: """Refuse to overwrite any evidence from an earlier live execution.""" occupied = [ @@ -473,17 +504,8 @@ def execute_audit_run( complete = False incomplete_reason = "audit_cost_limit_exhausted" break - prompt_path = output_directory / prompt["path"] - prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) - prompt_text = str(prompt_data["prompt_text"]) + prompt_text = _verified_prompt_text(output_directory, plan, prompt) name = str(prompt["name"]) - if ( - hashlib.sha256(prompt_text.encode()).hexdigest() - != plan["prompt_hashes"][name] - ): - raise AuditExecutionError( - f"Prompt content drift detected for {name}." - ) attempts = [] final_result: dict[str, Any] | None = None budget_exhausted = False diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index c1c21bc..33a758b 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -5,6 +5,7 @@ import os import sys from collections.abc import Callable, Sequence +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -41,6 +42,15 @@ CommandHandler = Callable[[argparse.Namespace], int] +@dataclass(frozen=True) +class RequiredRunArtifact: + """One conventional report input and its optional recovery review kind.""" + + label: str + path: Path + review_kind: str | None = None + + def build_parser() -> argparse.ArgumentParser: """Build the discoverable evaluation command hierarchy.""" parser = argparse.ArgumentParser( @@ -283,10 +293,9 @@ def _audit(arguments: argparse.Namespace) -> int: ensure_live_destination_available, execute_audit_run, load_verified_audit_plan, + normalize_audit_cost, ) - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() if arguments.auto_approve and not arguments.live: raise PrivateInputError("--auto-approve is accepted only with --live.") if arguments.live: @@ -327,6 +336,8 @@ def _audit(arguments: argparse.Namespace) -> int: return 0 if arguments.run_dir is not None: raise PrivateInputError("--run-dir is accepted only with --live.") + workspace = PrivateWorkspace.from_current_directory() + workspace.require_initialized() reference_path = workspace.root / "references" / "approved.json" snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" snapshot_path = snapshot_directory / "source.html" @@ -352,6 +363,7 @@ def _audit(arguments: argparse.Namespace) -> int: "Audit planning requires --max-audit-cost-usd as a positive " "spending guardrail." ) + maximum_cost = normalize_audit_cost(arguments.max_audit_cost_usd) eligibility_identity = reference_set_identity(reference_set) run_id, run_directory = workspace.create_evaluation_run() frozen_reference_path = run_directory / "reference-set.json" @@ -359,7 +371,7 @@ def _audit(arguments: argparse.Namespace) -> int: plan = build_audit_plan( snapshot_path, run_directory, - maximum_audit_cost_usd=arguments.max_audit_cost_usd, + maximum_audit_cost_usd=maximum_cost, reference_set_path=frozen_reference_path, workbook_sha256=reference_set.workbook_sha256, snapshot_sha256=snapshot_metadata["sha256"], @@ -381,9 +393,9 @@ def _audit(arguments: argparse.Namespace) -> int: def _review(arguments: argparse.Namespace) -> int: """Expose the private review workflow shell.""" - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() if arguments.eligibility_workbook: + workspace = PrivateWorkspace.from_current_directory() + workspace.require_initialized() reference_path = arguments.reference_set or ( workspace.root / "references" / "imported.json" ) @@ -411,10 +423,11 @@ def _review(arguments: argparse.Namespace) -> int: "Match review requires --run-dir and --kind " "(audit or programmatic)." ) - from vision_aid.evaluation.audit import load_verified_audit_plan + from vision_aid.evaluation.audit import load_audit_plan run_directory = arguments.run_dir.resolve() - plan = load_verified_audit_plan(run_directory) + workspace = PrivateWorkspace.from_evaluation_run(run_directory) + plan = load_audit_plan(run_directory) run_id = str(plan["run_id"]) review_directory = workspace.root / "reviews" / run_id reference_path = arguments.reference_set or ( @@ -447,6 +460,8 @@ def _review(arguments: argparse.Namespace) -> int: export_match_workbook(references, findings, workbook_path) print(f"Matches review workbook: {workbook_path.resolve()}") return 0 + workspace = PrivateWorkspace.from_current_directory() + workspace.require_initialized() print( f"Private review workspace: {(workspace.root / 'reviews').resolve()}" ) @@ -455,41 +470,54 @@ def _review(arguments: argparse.Namespace) -> int: def _report(arguments: argparse.Namespace) -> int: """Discover verified run evidence and write every report projection.""" - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() - from vision_aid.evaluation.audit import load_verified_audit_plan + from vision_aid.evaluation.audit import load_audit_plan run_directory = arguments.run_dir.resolve() - plan = load_verified_audit_plan(run_directory) + workspace = PrivateWorkspace.from_evaluation_run(run_directory) + plan = load_audit_plan(run_directory) run_id = str(plan["run_id"]) review_directory = workspace.root / "reviews" / run_id - inputs = { - "frozen reference set": run_directory / "reference-set.json", - "audit findings": run_directory / "canonical-audit-findings.json", - "programmatic findings": run_directory / "canonical-programmatic.json", - "audit match decisions": ( - review_directory / "audit-match-decisions.json" + artifacts = ( + RequiredRunArtifact( + "frozen reference set", run_directory / "reference-set.json" + ), + RequiredRunArtifact( + "audit findings", run_directory / "canonical-audit-findings.json" ), - "programmatic match decisions": ( - review_directory / "programmatic-match-decisions.json" + RequiredRunArtifact( + "programmatic findings", + run_directory / "canonical-programmatic.json", ), - "live manifest": run_directory / "live-manifest.json", - } - for label, path in inputs.items(): - if path.is_file(): + RequiredRunArtifact( + "audit match decisions", + review_directory / "audit-match-decisions.json", + "audit", + ), + RequiredRunArtifact( + "programmatic match decisions", + review_directory / "programmatic-match-decisions.json", + "programmatic", + ), + RequiredRunArtifact( + "live manifest", run_directory / "live-manifest.json" + ), + ) + for artifact in artifacts: + if artifact.path.is_file(): continue - if "match decisions" in label: - kind = label.split()[0] + if artifact.review_kind is not None: raise PrivateInputError( - f"Missing {label} at {path}. Run " + f"Missing {artifact.label} at {artifact.path}. Run " f"'visionaid-evaluate review --run-dir {run_directory} " - f"--kind {kind}', complete the workbook, then rerun with " + f"--kind {artifact.review_kind}', complete the workbook, " + "then rerun with " "--import-matches." ) raise PrivateInputError( - f"Missing {label} at {path}; complete the evaluation run before " - "reporting." + f"Missing {artifact.label} at {artifact.path}; complete the " + "evaluation run before reporting." ) + inputs = {artifact.label: artifact.path for artifact in artifacts} reference_set = load_reference_set(inputs["frozen reference set"]) live_manifest = json.loads( inputs["live manifest"].read_text(encoding="utf-8") diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py index 3fd0898..e6a9ff4 100644 --- a/vision_aid/evaluation/workspace.py +++ b/vision_aid/evaluation/workspace.py @@ -33,6 +33,19 @@ def from_current_directory(cls) -> "PrivateWorkspace": worktree = find_worktree_root(current_directory) return cls((worktree or current_directory) / DEFAULT_WORKSPACE) + @classmethod + def from_evaluation_run(cls, run_directory: Path) -> "PrivateWorkspace": + """Recover the private workspace associated with one planned run.""" + resolved_run = run_directory.resolve() + if resolved_run.parent.name != "runs": + raise PrivateInputError( + f"Evaluation run {resolved_run} is not inside a runs " + "partition." + ) + workspace = cls(resolved_run.parent.parent) + workspace.require_initialized() + return workspace + @property def partitions(self) -> tuple[Path, ...]: """Return every required private artifact partition.""" From 442c4cb91b4dbae649450c15eb2ba614cdf454fe Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 18:39:01 -0400 Subject: [PATCH 022/126] docs(evaluation): resolve run-oriented CLI workflow --- .scratch/evaluation-cli-workflow/spec.md | 126 ++++++++++++++++++ .../issues/07-safe-luna-dry-run.md | 11 +- .../issues/08-authorized-live-audit.md | 14 +- .../issues/09-normalize-raw-audit-findings.md | 2 +- .../issues/10-review-finding-matches.md | 6 +- .../11-verify-complete-evaluation-workflow.md | 10 +- .../issues/12-run-luna-homepage-poc.md | 12 +- 7 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 .scratch/evaluation-cli-workflow/spec.md diff --git a/.scratch/evaluation-cli-workflow/spec.md b/.scratch/evaluation-cli-workflow/spec.md new file mode 100644 index 0000000..52a78c2 --- /dev/null +++ b/.scratch/evaluation-cli-workflow/spec.md @@ -0,0 +1,126 @@ +# Streamline the Private Evaluation CLI Workflow + +Status: resolved + +## Problem Statement + +The private model-evaluation framework has the required safety and reproducibility controls, but its operator workflow is harder than necessary. A live run currently requires copying an authorization digest from a dry run into a later command even though the digest does not establish identity or provide meaningful consent. Match-review imports for audit findings and programmatic findings overwrite the same decision file, and final reporting requires manually constructing a report-bundle JSON document. The resulting walkthrough includes file renames and raw Python that should not be part of an ordinary evaluation run. + +The framework must remain dry by default, preserve its cost guardrail, verify that planned evidence has not drifted, and obtain explicit live-run approval. Those protections should be expressed through a clear point-of-use confirmation and one run-oriented CLI workflow rather than opaque copy-and-paste tokens. + +## Solution + +Make the evaluation run the operator-facing unit of work. Planning creates a unique run directory, records the maximum audit cost, freezes the approved reference set, and prints the billable intent. Live execution receives that run directory, revalidates its internal identities, prints the complete execution summary, and asks an interactive operator to type `yes`. Automation may deliberately replace the prompt with `--auto-approve`, but it receives the same summary and all other spending gates remain mandatory. + +Use the run identifier to organize separate audit and programmatic match-review artifacts. Let reporting discover its verified inputs from the run directory and associated run-scoped reviews. Remove the authorization-digest, plan-path, and report-bundle interfaces so the normal workflow requires neither manual renaming nor custom Python. + +## User Stories + +1. As an evaluation operator, I want a dry run to remain the default, so that an available API key cannot spend money by itself. +2. As an evaluation operator, I want live execution to retain an explicit `--live` flag, so that billable intent is visible in the command. +3. As a budget owner, I want every evaluation run planned with a positive maximum audit cost, so that spending has a defined upper guardrail. +4. As a budget owner, I want the planned cost guardrail frozen with the evaluation run, so that it cannot drift at execution time. +5. As an evaluation operator, I want the live command to reuse the saved cost guardrail, so that I do not need to repeat matching values across commands. +6. As an evaluation operator, I want the complete live execution summary shown immediately before approval, so that I understand what will run and what it may cost. +7. As an evaluation operator, I want the summary to identify the model, endpoint, and reasoning effort, so that model configuration is visible before spending. +8. As an evaluation operator, I want the summary to identify the benchmark snapshot and approved reference set, so that I know which evidence will be evaluated. +9. As an evaluation operator, I want the summary to list the prompt request set and request count, so that approval covers known work. +10. As an evaluation operator, I want the summary to show estimated input usage, retry policy, maximum audit cost, and destination, so that operational consequences are explicit. +11. As an evaluation operator, I want to approve a live run by typing `yes`, so that approval is understandable without interpreting a digest. +12. As an evaluation operator, I want any response other than an exact case-insensitive `yes` to abort, so that ambiguous input cannot start a provider request. +13. As an evaluation operator, I want end-of-input to abort before a provider client is created, so that a missing response cannot be treated as consent. +14. As an evaluation operator, I want interactive approval to require a terminal, so that scripts cannot silently pipe confirmation into the prompt. +15. As an automation author, I want an explicit `--auto-approve` option, so that deliberate non-interactive execution is supported. +16. As an automation author, I want `--auto-approve` accepted only for live execution, so that the option has one clear meaning. +17. As an automation author, I want the full execution summary printed even when approval is automatic, so that logs retain the approved intent. +18. As an auditor, I want the live manifest to record whether approval was interactive or automatic and when it occurred, so that approval provenance is visible without claiming a person's identity. +19. As an evaluation operator, I want live execution to require an API key in addition to approval, so that confirmation alone cannot create a request. +20. As a maintainer, I want internal plan and evidence identities retained, so that modified snapshots, prompts, pricing, eligibility decisions, or configurations still fail closed. +21. As an evaluation operator, I do not want to copy an authorization digest between commands, so that the workflow reflects meaningful consent rather than token matching. +22. As an evaluation operator, I want every planned evaluation run to receive a unique run directory, so that repeated experiments do not collide. +23. As an auditor, I want each evaluation run to freeze the approved reference set it used, so that later curation cannot make old evidence unreportable. +24. As an auditor, I want an evaluation run to retain its immutable model, benchmark, prompt, parser, pricing, and reference identities, so that reports remain reproducible. +25. As an evaluation operator, I want execution to refuse an occupied live-output destination, so that earlier evidence is never overwritten. +26. As an evaluation operator, I want a rerun to require a fresh evaluation run, so that each approval and result remain independently traceable. +27. As a comparison reviewer, I want audit findings and programmatic findings reviewed as distinct kinds, so that their match decisions cannot overwrite one another. +28. As a comparison reviewer, I want match workbooks and validated match decisions scoped by run identifier, so that reviews from different evaluations cannot collide. +29. As a comparison reviewer, I want the review command to infer conventional inputs and outputs from a run directory and review kind, so that ordinary review does not require repeated paths. +30. As a comparison reviewer, I want explicit input and output overrides retained for unusual review cases, so that the simplified workflow does not prevent expert recovery or investigation. +31. As an evaluation operator, I want the report command to accept one run directory, so that I do not manually enumerate scoring inputs. +32. As an evaluation operator, I want reporting to discover the frozen reference set, audit findings, programmatic findings, both match-decision sets, and live manifest, so that the correct evidence is assembled consistently. +33. As an evaluation operator, I want format failures read from the live manifest automatically, so that I do not duplicate run metadata in another document. +34. As an evaluation operator, I want missing or unresolved review artifacts reported with an actionable next command, so that the workflow explains how to proceed. +35. As an evaluation operator, I want JSON, CSV, and Markdown reports produced without a report bundle, so that no raw Python is required. +36. As a maintainer, I want obsolete authorization, plan-path, and report-bundle options removed immediately, so that the pre-proof-of-concept CLI has one supported workflow. +37. As a maintainer, I want the broader evaluation documentation updated to use live-run approval terminology, so that the design no longer implies that a digest authenticates an operator. +38. As a maintainer, I want the synthetic workflow to cover both audit and programmatic review decisions, so that the tested lifecycle matches the real reporting requirements. +39. As a project lead, I want these ergonomics changes kept separate from the actual Luna proof of concept, so that improving the CLI does not falsely mark the live evaluation complete. + +## Implementation Decisions + +- An evaluation run is the primary CLI concept joining one saved audit plan with its live evidence, reviewed match decisions, and reports. +- Dry audit planning automatically creates a unique run directory and prints its location for subsequent commands. +- The maximum audit-cost guardrail is mandatory during planning, validated as a positive decimal, and saved as part of the immutable plan. +- Live execution accepts the run directory rather than a separate plan path. It reads the saved guardrail and does not accept a replacement live value. +- Live execution continues to require the explicit live flag and an available OpenAI API key. +- The user-facing authorization digest, authorization option, one-use authorization receipt, authorized-plan duplicate field, and authorization-digest manifest field are removed. +- Internal content identities remain. Live execution revalidates the plan itself, frozen configuration, benchmark snapshot, approved eligibility decisions, prompt content, parser identity, pricing identity, and cost guardrail before any provider request. +- The complete billable-intent summary is rendered at live invocation before client construction. It includes model configuration, benchmark and reference identities, prompt names and count, estimated input tokens, retry policy, maximum audit cost, run directory, and approval mode. +- Interactive approval accepts only trimmed, case-insensitive `yes`. Any other response, end-of-input, or unavailable interactive terminal exits nonzero without constructing the provider client or making a request. +- Automatic approval is enabled only by an explicit `--auto-approve` option used together with live execution. It skips input but not summary rendering or any other gate. +- The live manifest records approval mode as `interactive` or `auto` plus an approval timestamp. It does not represent API-key presence as approval and does not claim a reviewer identity. +- Planning freezes a private copy of the approved reference set as run evidence. Scoring and reporting use that copy instead of the mutable current reference set. +- Live output paths are write-once. Existing live evidence causes an actionable failure that directs the operator to create a new evaluation run. +- Run-scoped match review distinguishes exactly two finding kinds: audit and programmatic. Each kind has a distinct workbook and validated match-decision artifact under a review partition keyed by run identifier. +- The review workflow accepts a run directory and finding kind, infers the conventional frozen reference set, findings input, review workbook, and match-decision output, and retains explicit overrides for exceptional workflows. +- Import mode continues to validate reviewer provenance, decision states, compatible evidence, required sub-defects, and one-to-one matching before writing decisions. +- Reporting accepts a run directory as its ordinary and only operator-facing input selector. It discovers all score inputs and obtains format failures directly from the live manifest. +- The report-bundle input and bundle-file format are removed from the CLI rather than retained as an advanced compatibility path. Tests and unconventional integrations can call the deterministic scorer interface directly. +- The obsolete plan-path option is removed because the plan has a conventional location within its evaluation run. +- Report formats, scoring rules, retry policy, prompt content, model configuration, normalization, matching semantics, and ranking behavior do not change. +- Documentation and the parent model-evaluation specification are revised to replace authorization-digest instructions with live-run approval and the run-oriented commands. +- Existing resolved implementation tickets remain resolved after their acceptance criteria and comments are made accurate. The operational Luna proof-of-concept ticket remains open until its private preparation, live execution, reviews, and final reports are actually completed. +- The domain glossary defines evaluation run and live-run approval. No architecture decision record is required because this pre-proof-of-concept interface change is straightforward to reverse and is not an architectural lock-in. + +## Testing Decisions + +- Tests assert externally visible behavior rather than private confirmation helpers, naming functions, or filesystem implementation order. +- The primary test seam is the evaluation CLI entry point exercised through a complete synthetic lifecycle in a temporary private workspace with a fake provider. +- The end-to-end CLI test plans a uniquely identified evaluation run, verifies the frozen reference set and saved cost limit, executes it with controlled approval, reviews audit findings, reviews programmatic findings, and generates all three reports without a report bundle. +- Focused CLI tests verify interactive acceptance, case and whitespace handling, rejection, end-of-input, unavailable terminal behavior, and automatic approval. +- Approval failure tests assert that no provider client is constructed, no request is made, and no live artifacts are created. +- Safety tests verify that the live flag, API key, saved positive cost guardrail, exact plan integrity, and frozen identities remain mandatory. +- Overwrite tests execute against occupied live destinations and assert that existing bytes remain unchanged and the command directs the operator to create a fresh evaluation run. +- Review tests prove that audit and programmatic decision outputs are distinct, run-scoped, validated, discoverable, and both required for final reporting. +- Reporting tests invoke only the run-directory workflow and verify automatic discovery of the frozen reference set, both findings collections, both match-decision sets, manifest metadata, and format failures. +- Parser tests verify that obsolete authorization, plan-path, and report-bundle options are rejected rather than silently ignored. +- Lower-level audit-executor tests remain for retry classification, cost accumulation, exhausted budgets, malformed successful responses, and immediate attempt persistence where those behaviors are clearer below the CLI seam. +- Existing synthetic evaluation CLI tests are the prior art. They will be reshaped around the run-oriented lifecycle rather than supplemented with a second parallel workflow. +- No test uses the private workbook, live Pristine content, a real API key, public network access, or billable provider calls. + +## Out of Scope + +- Performing the real Luna homepage proof of concept or checking off its operational acceptance criteria. +- Changing the GPT-5.6 Luna model, medium reasoning effort, endpoint, output limit, prompt order, filters, summaries, or retry policy. +- Changing eligibility classifications, match semantics, scoring rules, ranking metrics, report contents, or the illustrative-result limitation. +- Adding Astra-backed eligibility or match reviewers. +- Adding a web interface or public API for evaluation runs. +- Supporting report bundles as a hidden, deprecated, or advanced CLI feature. +- Authenticating the human who types `yes` or treating live-run approval as a cryptographic signature. +- Allowing approval to bypass API credentials, integrity checks, the cost guardrail, or overwrite protection. +- Migrating previously completed private evaluation artifacts; no real live proof-of-concept run exists yet. + +## Further Notes + +- The authorization digest was a confirmation checksum, not an identity proof. Point-of-use approval provides clearer operator intent while internal hashes continue to provide drift detection. +- A provider API key is a credential, not live-run approval. Its presence never triggers spending without the live flag and approval. +- A separate invocation, including one using automatic approval, constitutes a new live-run approval. A materially different configuration requires a newly planned evaluation run. +- The run directory is the operator's handle for live execution, review, and reporting even though private artifacts may remain partitioned by responsibility within the evaluation workspace. +- The existing private workbook and its SHA-256 remain unchanged by this feature. + +## Comments + +Implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by the +synthetic CLI lifecycle, focused approval and identity-drift tests, lower-level +retry and cost tests, compilation, and the full network-free pytest suite. The +operational Luna proof of concept remains open and no provider request was made. diff --git a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md index e846876..a9b6dda 100644 --- a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md +++ b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md @@ -1,6 +1,6 @@ # 07: Prepare a safe Luna dry run -**What to build:** Produce a no-cost Luna audit plan against the approved benchmark configuration, with complete provenance and an explicit summary suitable for spending authorization. +**What to build:** Produce a no-cost Luna audit plan against the approved benchmark configuration, with complete provenance and an explicit summary suitable for point-of-use live-run approval. **Blocked by:** 01: Make audit request configuration explicit; 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot; 04: Generate and validate the homepage reference set. @@ -10,11 +10,14 @@ - [x] The planned configuration records GPT-5.6 Luna, medium reasoning, Chat Completions, and omitted temperature. - [x] Prompt payloads, request count, estimated input tokens, prompt hashes, parser hashes, and repository state are saved without making a provider call. - [x] The manifest records workbook, snapshot, reference-set, benchmark, configuration, and pricing identities. -- [x] The command refuses live execution when the live flag, API key, approved reference set, or maximum audit-cost limit is absent. +- [x] Planning creates a unique evaluation run, freezes the approved reference set, and requires a positive maximum audit-cost limit. +- [x] The command refuses live execution when the live flag, API key, saved positive cost guardrail, exact planned evidence, or live-run approval is absent. - [x] Tests prove that an environment key alone cannot trigger spending. ## Comments Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Verified by dry-run manifest and no-spend gate tests; no provider -call is made by the test suite. +and identity-verified in `2129616`. The run-oriented planning workflow was +implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by unique +run, frozen-evidence, positive-cost, identity-drift, and no-spend tests; no +provider call is made by the test suite. diff --git a/.scratch/model-evaluation/issues/08-authorized-live-audit.md b/.scratch/model-evaluation/issues/08-authorized-live-audit.md index 24c92ec..955e775 100644 --- a/.scratch/model-evaluation/issues/08-authorized-live-audit.md +++ b/.scratch/model-evaluation/issues/08-authorized-live-audit.md @@ -1,22 +1,26 @@ -# 08: Execute and record an authorized live audit +# 08: Execute and record an approved live audit -**What to build:** Add the guarded live-audit path that executes an explicitly authorized Luna run, preserves every raw outcome, and accounts for reliability, usage, cost, and latency. +**What to build:** Add the guarded live-audit path that executes a point-of-use approved Luna run, preserves every raw outcome, and accounts for reliability, usage, cost, and latency. **Blocked by:** 01: Make audit request configuration explicit; 07: Prepare a safe Luna dry run. **Status:** resolved -- [x] Live execution requires the explicit live flag, API key, maximum audit-cost limit, and the exact summarized benchmark configuration. +- [x] Live execution requires the explicit live flag, API key, saved maximum audit-cost limit, exact summarized benchmark configuration, and interactive or automatic approval. - [x] Audit prompts run sequentially with medium reasoning, no temperature, disabled summaries, and the existing output limit. - [x] At most two additional attempts occur for timeouts, rate limits, and server errors using bounded exponential backoff. - [x] Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. - [x] Every attempt records request identity, response metadata, usage, duration, stop reason, and failure classification without exposing secrets. - [x] Billed retry usage contributes to audit cost, and the runner stops before starting another request once the cost guardrail is exhausted. - [x] Exhausted transient failures mark the run incomplete and unranked; malformed successful responses remain recorded as model-format failures. +- [x] The live manifest records approval mode and timestamp without claiming operator identity. +- [x] Existing live evidence is never overwritten; another execution requires a newly planned evaluation run. - [x] Fake-provider tests exercise success, retry, budget, malformed response, and incomplete-run behavior without spending money. ## Comments Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Fake-provider tests verify authorization, retry, checkpoint, -budget, malformed-response, and incomplete-run policies without spending. +and identity-verified in `2129616`. Point-of-use approval and write-once run +execution were implemented in `2448042` and copy/rerun isolation was hardened +in `6a1e6c0`. Fake-provider tests verify approval, retry, checkpoint, budget, +malformed-response, and incomplete-run policies without spending. diff --git a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md index eba1aa6..ac0b0a5 100644 --- a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md +++ b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md @@ -2,7 +2,7 @@ **What to build:** Convert saved raw audit responses into lossless canonical audit findings without relying on the production CSV, suppression rules, or model-based deduplication. -**Blocked by:** 08: Execute and record an authorized live audit. +**Blocked by:** 08: Execute and record an approved live audit. **Status:** resolved diff --git a/.scratch/model-evaluation/issues/10-review-finding-matches.md b/.scratch/model-evaluation/issues/10-review-finding-matches.md index 8551883..fd472f9 100644 --- a/.scratch/model-evaluation/issues/10-review-finding-matches.md +++ b/.scratch/model-evaluation/issues/10-review-finding-matches.md @@ -13,9 +13,13 @@ - [x] WCAG identifiers and remediation wording remain supporting evidence rather than exact-match requirements. - [x] Approved decisions round-trip through the `MatchReviewer` interface and enforce one-to-one matching. - [x] Conflicts, unresolved decisions, incomplete grouped evidence, and missing required sub-defects fail clearly rather than being resolved silently. +- [x] Audit and programmatic findings use distinct run-scoped workbooks and validated decision artifacts. +- [x] The CLI infers conventional review paths from the evaluation run and finding kind while retaining explicit overrides. ## Comments Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Candidate generation, workbook round-trip, evidence validation, +and identity-verified in `2129616`. Run-scoped audit and programmatic review +discovery was implemented in `2448042` and historical-run loading was hardened +in `6a1e6c0`. Candidate generation, workbook round-trip, evidence validation, and conflict tests pass in the synthetic suite. diff --git a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md index 91d498b..ffab7b0 100644 --- a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md +++ b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md @@ -2,7 +2,7 @@ **What to build:** Connect preparation, audit, review, scoring, and reporting into one coherent workflow and prove the entire lifecycle with synthetic provider data before any real model run. -**Blocked by:** 05: Review and score a synthetic comparison; 06: Measure programmatic workbook coverage; 08: Execute and record an authorized live audit; 10: Generate and validate finding matches. +**Blocked by:** 05: Review and score a synthetic comparison; 06: Measure programmatic workbook coverage; 08: Execute and record an approved live audit; 10: Generate and validate finding matches. **Status:** resolved @@ -13,10 +13,12 @@ - [x] End-to-end and per-request latency are reported but never ranked. - [x] JSON, CSV, and Markdown reports expose every in-scope workbook row and agree on scoring and coverage. - [x] CI runs the pytest suite without private workbook data, public network access, provider credentials, or spending. -- [x] Documentation explains the private-data workflow, human approval gates, dry-run review, live authorization, and illustrative limits of the homepage benchmark. +- [x] Documentation explains the private-data workflow, evaluation-run planning, point-of-use live-run approval, run-scoped reviews, discovered reporting inputs, and illustrative limits of the homepage benchmark. ## Comments Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. The 55-test suite proves the synthetic CLI lifecycle and failure -policies; mypy, Ruff, lock validation, and the production dry run also pass. +and identity-verified in `2129616`. The run-oriented synthetic lifecycle and +documentation were implemented in `2448042` and review-hardened in `6a1e6c0`. +The suite proves both review kinds, discovered reporting, approval failure +paths, immutable identities, and write-once execution without provider access. diff --git a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md index 285288f..f7cae14 100644 --- a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md +++ b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md @@ -7,10 +7,16 @@ **Status:** ready-for-agent - [ ] The private workbook checksum, snapshot, reference set, and eligibility classifications are reviewed and approved before the audit proceeds. -- [ ] A no-cost dry-run summary presents the exact Luna configuration, request set, estimated usage, and proposed cost limit. -- [ ] The live run begins only after explicit user authorization covering that summarized configuration and cost guardrail. -- [ ] The authorized Luna run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. +- [ ] A no-cost evaluation-run summary presents the exact Luna configuration, request set, estimated usage, and planned cost limit. +- [ ] The live run begins only after point-of-use approval covering the displayed configuration and saved cost guardrail. +- [ ] The approved Luna run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. - [ ] Canonical findings and candidate matches are generated without production CSV filtering or model-based deduplication. - [ ] Ambiguous match decisions receive human approval before scoring. - [ ] The final reports show Luna workbook-row recall, programmatic coverage, combined workbook coverage, estimated audit cost, tokens, latency, unmatched findings, and format failures. - [ ] The result is labeled illustrative and makes no small-versus-large model claim from the homepage's limited eligible denominator. + +## Comments + +The run-oriented CLI ergonomics were implemented in `2448042` and hardened in +`6a1e6c0`. This operational ticket remains open: no private preparation, live +Luna request, human match review, or final proof-of-concept report was performed. From 210044c90775bbf5026412ecc08e7013ed35d4d8 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 19 Sep 2026 22:08:52 -0400 Subject: [PATCH 023/126] fix(evaluation): map mandated workbook headers --- tests/test_reference_set.py | 43 +++++++++++++++++++++++++++++ vision_aid/evaluation/references.py | 25 ++++++++++++++--- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/tests/test_reference_set.py b/tests/test_reference_set.py index 88fc89a..b36b976 100644 --- a/tests/test_reference_set.py +++ b/tests/test_reference_set.py @@ -94,6 +94,49 @@ def test_import_homepage_references_retains_every_home_and_global_source_row( ) +def test_import_homepage_references_maps_mandated_workbook_headers( + tmp_path: Path, +) -> None: + """The mandated workbook's field names populate canonical evidence.""" + workbook_path = tmp_path / "mandated-headers.xlsx" + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Defect report" + sheet.append( + [ + "Page name", + "Issue Title", + "element name", + "WCAG Sc", + "Recommendation for Fix", + ] + ) + sheet.append( + [ + "Home", + "No main landmark defined on the page", + "Region", + "1.3.1", + "Use a main element.", + ] + ) + workbook.save(workbook_path) + checksum = hashlib.sha256(workbook_path.read_bytes()).hexdigest() + + reference_set = import_homepage_references( + workbook_path, + expected_sha256=checksum, + homepage_url="https://example.test/", + version="synthetic-v1", + ) + + reference = reference_set.references[0] + assert reference.problem == "No main landmark defined on the page" + assert reference.location == "Region" + assert reference.wcag_evidence == ("1.3.1",) + assert reference.recommendation == "Use a main element." + + def test_human_eligibility_review_round_trips_validated_provenance( tmp_path: Path, ) -> None: diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/references.py index e6718c8..602f30e 100644 --- a/vision_aid/evaluation/references.py +++ b/vision_aid/evaluation/references.py @@ -34,10 +34,27 @@ def review(self, reference_set: ReferenceSet) -> ReferenceSet: _ALIASES = { "scope": ("page", "page scope", "scope", "page name"), "url": ("url", "page url", "web address"), - "problem": ("problem", "issue", "defect", "description"), - "location": ("location", "element", "where"), - "wcag": ("wcag", "wcag criteria", "success criterion", "criterion"), - "recommendation": ("recommendation", "remediation", "fix"), + "problem": ( + "problem", + "issue", + "issue title", + "defect", + "description", + ), + "location": ("location", "element", "element name", "where"), + "wcag": ( + "wcag", + "wcag sc", + "wcag criteria", + "success criterion", + "criterion", + ), + "recommendation": ( + "recommendation", + "recommendation for fix", + "remediation", + "fix", + ), } ELIGIBILITY_VALUES = ( "llm_eligible", From 0b9c3d2f2de012b00153e2696316dd39296e7bbf Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 20 Sep 2026 11:32:46 -0400 Subject: [PATCH 024/126] refactor(evaluation): retire legacy audit runner --- README.md | 9 +- processing_scripts/llm_client/__init__.py | 25 +- processing_scripts/llm_client/client.py | 266 +----------------- .../llm_client/prompt_loader.py | 72 ----- processing_scripts/llm_client/runner.py | 159 ----------- tests/test_evaluation_cli.py | 24 ++ vision_aid/evaluation/cli.py | 15 + 7 files changed, 45 insertions(+), 525 deletions(-) delete mode 100644 processing_scripts/llm_client/prompt_loader.py delete mode 100644 processing_scripts/llm_client/runner.py diff --git a/README.md b/README.md index 537ddd8..5c3436c 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,9 @@ HTML file (e.g. 1.9 MB) │ │ ├── forms_checklist_02.txt # 6 prompts for form accessibility │ │ └── nontext_checklist_03.txt # 8 prompts for non-text content │ │ -│ ├── llm_client/ # Standalone Claude API client (Andrew) -│ │ ├── client.py # API wrapper -│ │ ├── prompt_loader.py # Prompt loading utilities -│ │ └── runner.py # End-to-end audit runner +│ ├── llm_client/ # Provider-independent audit requests +│ │ ├── audit.py # API requests and response metadata +│ │ └── client.py # Model-provider capability helpers │ │ │ ├── llm_preprocessing/ # HTML → structured JSON extractors │ │ ├── semantic_checklist_01.py # Headings, links, landmarks, tables, iframes @@ -100,7 +99,7 @@ HTML file (e.g. 1.9 MB) ├── docs/ # Architecture documentation │ └── modular-prompts-plan.md # Full architectural plan │ -├── reports/ # Raw JSON output from standalone llm_client runs +├── reports/ # Historical raw JSON audit output │ ├── test_results/ │ ├── chatgpt/ # Legacy ChatGPT testing results diff --git a/processing_scripts/llm_client/__init__.py b/processing_scripts/llm_client/__init__.py index 7fa4c70..192290d 100644 --- a/processing_scripts/llm_client/__init__.py +++ b/processing_scripts/llm_client/__init__.py @@ -1,31 +1,8 @@ -""" -llm_client — LLM API clients for the VisionAid accessibility audit pipeline. - -Supports both Anthropic (Claude) and OpenAI (GPT) models. - -Usage ------ - from llm_client import create_audit_client, load_all_prompts, run_all - - client = create_audit_client(model="claude-sonnet-5") # or "gpt-4o" - prompts = load_all_prompts(PROMPTS_DIR) - report = run_all(client, prompts, all_slices) -""" +"""Provider-independent requests for the accessibility audit pipeline.""" from .audit import AuditRequestClient, AuditRequestConfig -from .client import AuditClient, OpenAIAuditClient, create_audit_client, is_openai_model -from .prompt_loader import load_prompts, load_all_prompts -from .runner import run_checklist, run_all __all__ = [ - "AuditClient", "AuditRequestClient", "AuditRequestConfig", - "OpenAIAuditClient", - "create_audit_client", - "is_openai_model", - "load_prompts", - "load_all_prompts", - "run_checklist", - "run_all", ] diff --git a/processing_scripts/llm_client/client.py b/processing_scripts/llm_client/client.py index 139f049..d4eecfc 100644 --- a/processing_scripts/llm_client/client.py +++ b/processing_scripts/llm_client/client.py @@ -1,18 +1,4 @@ -""" -AuditClient — thin wrappers around the Anthropic and OpenAI APIs. - -Loads API keys from the project root .env file automatically. -""" - -import json -import os -from pathlib import Path - -from dotenv import load_dotenv - -# Project root is two levels up from this file (processing_scripts/llm_client/) -_ENV_FILE = Path(__file__).resolve().parents[2] / ".env" -load_dotenv(_ENV_FILE) +"""Model-provider detection and request capability helpers.""" # OpenAI model prefixes used to detect provider from model ID _OPENAI_PREFIXES = ("gpt-", "o1", "o3", "o4") @@ -48,253 +34,3 @@ def is_gemini_model(model: str) -> bool: def supports_temperature(model: str) -> bool: """Return False for Claude models that reject the `temperature` param.""" return model not in _NO_SAMPLING_PARAMS_MODELS - - -class AuditClient: - """ - Sends filled accessibility-audit prompts to the Claude API and returns - parsed JSON responses. - - Parameters - ---------- - model : str - Claude model ID to use (default: claude-sonnet-5). Also accepts - ``claude-opus-5`` and ``claude-fable-5``, Anthropic's most capable - model. Newer Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) - reject a ``temperature`` param entirely (see ``supports_temperature``), - which this client already accounts for. - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - Silently omitted from the request for models that reject it. - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "claude-sonnet-5", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("ANTHROPIC_API_KEY") - if not api_key: - raise OSError( - f"ANTHROPIC_API_KEY not found. Expected in {_ENV_FILE}" - ) - import anthropic - - self._client = anthropic.Anthropic(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - The response is expected to be a JSON object (possibly wrapped in a - markdown code fence, which is stripped automatically). - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - serialized_payload = json.dumps(payload, separators=(",", ":")) - filled = prompt_text.replace("{payload}", serialized_payload) - - request_kwargs = { - "model": self.model, - "max_tokens": self.max_tokens, - "messages": [{"role": "user", "content": filled}], - } - if supports_temperature(self.model): - request_kwargs["temperature"] = self.temperature - - message = self._client.messages.create(**request_kwargs) - - usage = { - "input_tokens": message.usage.input_tokens, - "output_tokens": message.usage.output_tokens, - } - - raw = message.content[0].text.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -class OpenAIAuditClient: - """ - Sends filled accessibility-audit prompts to the OpenAI API and returns - parsed JSON responses. Same interface as :class:`AuditClient`. - - Parameters - ---------- - model : str - OpenAI model ID to use (default: gpt-4o). - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "gpt-4o", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("OPENAI_API_KEY") - if not api_key: - raise OSError( - f"OPENAI_API_KEY not found. Expected in {_ENV_FILE}" - ) - import openai - - self._client = openai.OpenAI(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - serialized_payload = json.dumps(payload, separators=(",", ":")) - filled = prompt_text.replace("{payload}", serialized_payload) - - response = self._client.chat.completions.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=self.temperature, - messages=[{"role": "user", "content": filled}], - ) - - usage = { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - } - - raw = response.choices[0].message.content.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -class GeminiAuditClient: - """ - Sends filled accessibility-audit prompts to the Gemini API and returns - parsed JSON responses. Same interface as :class:`AuditClient`. - - Parameters - ---------- - model : str - Gemini model ID to use (default: gemini-flash-latest). Prefer the - ``-latest`` aliases over dated model IDs (e.g. ``gemini-2.5-flash``), - which Google periodically retires for new API keys. - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "gemini-flash-latest", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("GEMINI_API_KEY") - if not api_key: - raise OSError( - f"GEMINI_API_KEY not found. Expected in {_ENV_FILE}" - ) - from google import genai - - self._client = genai.Client(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - serialized_payload = json.dumps(payload, separators=(",", ":")) - filled = prompt_text.replace("{payload}", serialized_payload) - - response = self._client.models.generate_content( - model=self.model, - contents=filled, - config={ - "temperature": self.temperature, - "max_output_tokens": self.max_tokens, - }, - ) - - usage = { - "input_tokens": response.usage_metadata.prompt_token_count, - "output_tokens": response.usage_metadata.candidates_token_count, - } - - raw = response.text.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -def create_audit_client( - model: str = "claude-sonnet-5", - temperature: float = 0.1, - max_tokens: int = 8192, -) -> AuditClient | OpenAIAuditClient | GeminiAuditClient: - """Factory that returns the right client class based on the model ID.""" - if is_openai_model(model): - return OpenAIAuditClient( - model=model, - temperature=temperature, - max_tokens=max_tokens, - ) - if is_gemini_model(model): - return GeminiAuditClient( - model=model, - temperature=temperature, - max_tokens=max_tokens, - ) - return AuditClient( - model=model, - temperature=temperature, - max_tokens=max_tokens, - ) diff --git a/processing_scripts/llm_client/prompt_loader.py b/processing_scripts/llm_client/prompt_loader.py deleted file mode 100644 index 60d33d6..0000000 --- a/processing_scripts/llm_client/prompt_loader.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Parses the numbered prompt sections from the LLM prompt-library text files. - -File format (see processing_scripts/llm/*.txt): - --------------------------------- - N) PROMPT TITLE - --------------------------------- - - Data: {payload} - - --------------------------------- - N+1) NEXT PROMPT TITLE - ... -""" - -import re -from pathlib import Path - - -_HEADER_RE = re.compile(r"^\s{2,}(\d+)\)\s+.+", re.MULTILINE) -_DIVIDER_RE = re.compile(r"^\s*-{5,}\s*$", re.MULTILINE) - - -def load_prompts(filepath: str | Path) -> dict[int, str]: - """ - Parse *filepath* and return ``{prompt_number: prompt_text}`` for every - numbered section found. Divider lines (``---...``) are stripped. - - The returned text for each prompt still contains the ``{payload}`` - placeholder so the caller can fill it later. - """ - text = Path(filepath).read_text(encoding="utf-8") - - # Find all header positions and their prompt numbers - headers = list(_HEADER_RE.finditer(text)) - if not headers: - return {} - - prompts: dict[int, str] = {} - - for i, header in enumerate(headers): - num = int(header.group(1)) - start = header.end() # character just after the header line - end = headers[i + 1].start() if i + 1 < len(headers) else len(text) - - body = text[start:end] - - # Remove divider lines - body = _DIVIDER_RE.sub("", body) - - prompts[num] = body.strip() - - return prompts - - -def load_all_prompts(prompts_dir: str | Path) -> dict[str, dict[int, str]]: - """ - Load all ``*.txt`` prompt files in *prompts_dir*. - - Returns ``{stem: {prompt_number: prompt_text}}``, e.g.:: - - { - "semantic_checklist_01": {1: "...", 2: "...", ...}, - "forms_checklist_02": {1: "...", ...}, - "nontext_checklist_03": {1: "...", ...}, - } - """ - prompts_dir = Path(prompts_dir) - result: dict[str, dict[int, str]] = {} - for txt_file in sorted(prompts_dir.glob("*.txt")): - result[txt_file.stem] = load_prompts(txt_file) - return result diff --git a/processing_scripts/llm_client/runner.py b/processing_scripts/llm_client/runner.py deleted file mode 100644 index 97bbee8..0000000 --- a/processing_scripts/llm_client/runner.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -High-level runner that executes one or all checklist prompt sets against -pre-built JSON payloads, collects responses, and aggregates token usage. -""" - -from __future__ import annotations - -from typing import Any - -from .client import AuditClient - - -def run_checklist( - client: AuditClient, - prompts: dict[int, str], - slices: dict[str, Any | None], - *, - verbose: bool = True, -) -> dict[str, Any]: - """ - Run every prompt in *prompts* against the matching payload slice in *slices*. - - Parameters - ---------- - client : AuditClient - Configured API client. - prompts : dict[int, str] - ``{prompt_number: prompt_text}`` from ``load_prompts()``. - slices : dict[str, payload | None] - Mapping of label → payload dict (or ``None`` to skip). - Labels are used as keys in the returned results dict. - verbose : bool - Print progress to stdout while running. - - Returns - ------- - dict with keys: - - ``"results"`` : ``{label: response_json}`` - - ``"skipped"`` : list of skipped labels - - ``"usage"`` : ``{"input_tokens": int, "output_tokens": int}`` - - ``"errors"`` : ``{label: error_message}`` - """ - results: dict[str, Any] = {} - skipped: list[str] = [] - errors: dict[str, str] = {} - total_input = 0 - total_output = 0 - - # Map labels → prompt numbers by position (label order == prompt order) - labels = list(slices.keys()) - - for idx, label in enumerate(labels): - prompt_num = idx + 1 - payload = slices[label] - - if payload is None: - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (filtered by Pass 1)") - skipped.append(label) - continue - - # Skip if the payload is empty after filtering - if _is_empty_payload(payload): - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (empty payload after filtering)") - skipped.append(label) - continue - - prompt_text = prompts.get(prompt_num) - if prompt_text is None: - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (no prompt text found)") - skipped.append(label) - continue - - if verbose: - print(f" [{prompt_num}] {label} ...", end=" ", flush=True) - - try: - response, usage = client.call(prompt_text, payload) - results[label] = response - total_input += usage["input_tokens"] - total_output += usage["output_tokens"] - if verbose: - print(f"✓ ({usage['input_tokens']}in / {usage['output_tokens']}out tokens)") - except Exception as exc: - errors[label] = str(exc) - if verbose: - print(f"✗ ERROR: {exc}") - - return { - "results": results, - "skipped": skipped, - "usage": {"input_tokens": total_input, "output_tokens": total_output}, - "errors": errors, - } - - -def run_all( - client: AuditClient, - all_prompts: dict[str, dict[int, str]], - all_slices: dict[str, dict[str, Any | None]], - *, - verbose: bool = True, -) -> dict[str, Any]: - """ - Run all checklists in *all_slices* and return a consolidated report dict. - - Parameters - ---------- - client : AuditClient - all_prompts : dict[str, dict[int, str]] - ``{checklist_stem: {prompt_num: prompt_text}}`` from ``load_all_prompts()``. - all_slices : dict[str, dict[str, payload|None]] - ``{checklist_stem: {label: payload|None}}``. - verbose : bool - - Returns - ------- - dict with keys: - - ``"checklists"`` : ``{stem: checklist_result}`` - - ``"total_usage"`` : ``{"input_tokens": int, "output_tokens": int}`` - """ - checklists: dict[str, Any] = {} - grand_input = 0 - grand_output = 0 - - for stem, slices in all_slices.items(): - prompts = all_prompts.get(stem, {}) - if verbose: - print(f"\n--- {stem} ---") - - result = run_checklist(client, prompts, slices, verbose=verbose) - checklists[stem] = result - grand_input += result["usage"]["input_tokens"] - grand_output += result["usage"]["output_tokens"] - - return { - "checklists": checklists, - "total_usage": {"input_tokens": grand_input, "output_tokens": grand_output}, - } - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _is_empty_payload(payload: Any) -> bool: - """Return True if every list-valued entry in the payload is empty.""" - if not isinstance(payload, dict): - return False - for v in payload.values(): - if isinstance(v, list) and v: - return False - if isinstance(v, dict) and v: - return False - if isinstance(v, str) and v.strip(): - return False - return True diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index dcfccf0..adc8c0b 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -16,6 +16,7 @@ import pytest from vision_aid.evaluation import audit as evaluation_audit +from vision_aid.evaluation import cli as evaluation_cli from vision_aid.evaluation.cli import main as evaluation_main from vision_aid.evaluation.schemas import ReferenceSet from vision_aid.evaluation.serialization import save_reference_set @@ -121,6 +122,29 @@ def isatty(self) -> bool: return self.terminal +def test_audit_loads_the_project_env_explicitly( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The evaluation audit keeps local .env support without import effects.""" + loaded_paths: list[Path] = [] + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(evaluation_cli, "_PROJECT_ENV_LOADED", False) + monkeypatch.setattr( + evaluation_cli, + "load_dotenv", + lambda path: loaded_paths.append(Path(path)), + ) + + with pytest.raises(SystemExit) as exit_info: + evaluation_main(["audit"]) + with pytest.raises(SystemExit): + evaluation_main(["audit"]) + + assert exit_info.value.code == 2 + assert loaded_paths == [PROJECT_ROOT / ".env"] + + def create_planned_run( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 33a758b..341b481 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -9,6 +9,8 @@ from datetime import UTC, datetime from pathlib import Path +from dotenv import load_dotenv + from vision_aid.evaluation.matching import ( HumanMatchReviewer, MatchValidationError, @@ -40,6 +42,17 @@ ) CommandHandler = Callable[[argparse.Namespace], int] +PROJECT_ROOT = Path(__file__).resolve().parents[2] +_PROJECT_ENV_LOADED = False + + +def _load_project_env() -> None: + """Load local evaluation credentials once per CLI process.""" + global _PROJECT_ENV_LOADED + if _PROJECT_ENV_LOADED: + return + load_dotenv(PROJECT_ROOT / ".env") + _PROJECT_ENV_LOADED = True @dataclass(frozen=True) @@ -288,6 +301,8 @@ def _confirm_live_execution() -> str: def _audit(arguments: argparse.Namespace) -> int: """Plan a no-cost evaluation run or execute one after explicit approval.""" + _load_project_env() + from vision_aid.evaluation.audit import ( build_audit_plan, ensure_live_destination_available, From 6ac33110c78bb17b77e5fd40c52f2ef3e2294bb0 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 20 Sep 2026 12:25:44 -0400 Subject: [PATCH 025/126] docs(evaluation): add implementation progress report Document the CLI workflow, eligibility classifications, current Luna proof-of-concept status, and remaining work. Expand the domain glossary for the human-audit reference workbook and non-model eligibility classes. --- .scratch/model-evaluation/progress-report.md | 442 +++++++++++++++++++ CONTEXT.md | 16 + 2 files changed, 458 insertions(+) create mode 100644 .scratch/model-evaluation/progress-report.md diff --git a/.scratch/model-evaluation/progress-report.md b/.scratch/model-evaluation/progress-report.md new file mode 100644 index 0000000..85576ee --- /dev/null +++ b/.scratch/model-evaluation/progress-report.md @@ -0,0 +1,442 @@ +# Model evaluation suite progress report + +Updated: September 20, 2026 + +## Term used in this report + +**Reference workbook** means the authorized Excel file containing the findings +from the prior human accessibility audit. Its individual tabs are worksheets, +and their rows supply the source evidence for the benchmark's reference +defects. The generated Eligibility and Matches Excel files are review +workbooks, not the reference workbook. + +## Why the evaluation suite is separate + +- I built a separate evaluation suite because the production CSV is a cleaned + customer deliverable rather than a complete record of model performance. + +- The CSV has no normalizers for six registered non-summary prompt categories: + table semantics, placeholder-only labels, form-group labels, form + instructions, complex image descriptions, and media captions. + +- The CSV also skips failed or malformed responses, combines model and + programmatic findings, applies false-positive filters, and can use another + LLM for deduplication. + +- Scoring that output would measure the report-processing pipeline along with + the audit model. + +- The evaluation suite instead normalizes every raw prompt response, preserves + failures and provenance, separates programmatic coverage, and scores only + approved matches against eligible workbook rows. + +- I reused the production pipeline for prompt generation and provider requests + so the evaluation still tests the same audit behavior used by the + application. + +## CLI command order + +Run the following commands in order. Complete the Eligibility workbook before +command 2 and each generated Matches workbook before its following +`--import-matches` command. `RUN_DIR` is the evaluation-run path printed by +command 3. + +```bash +# 1. Build the benchmark from the human-audit reference workbook. +# This imports the reference defects and freezes the homepage HTML. +# --homepage-url identifies which workbook page rows belong to the +# benchmark. It selects the Home and applicable Global reference defects +# but does not fetch the webpage. +# --snapshot-url identifies where to download the HTML that models will +# inspect. It records the captured content and its provenance. +# Both values are the same for this proof of concept because the workbook +# homepage and the captured benchmark page are both pristineai.com. +uv run visionaid-evaluate prepare \ + --workbook-sha256 "$WORKBOOK_SHA256" \ + --homepage-url https://pristineai.com/ \ + --snapshot-url https://pristineai.com/ \ + --temporal-assumption "$TEMPORAL_ASSUMPTION" + +# Manually complete .model-evaluation/reviews/eligibility.xlsx. +# Classify who is responsible for each reference defect before scoring begins. + +# 2. Validate the completed eligibility decisions and create the approved +# reference set used by every evaluation run. +uv run visionaid-evaluate review \ + --eligibility-workbook .model-evaluation/reviews/eligibility.xlsx + +# 3. Plan the model evaluation without making provider requests. +# This freezes the prompts, model settings, evidence identities, and cost +# guardrail. Save the evaluation-run path printed by this command. +uv run visionaid-evaluate audit --max-audit-cost-usd 1.00 + +RUN_DIR=".model-evaluation/runs/REPLACE_WITH_PRINTED_RUN_ID" + +# 4. Generate a workbook that compares deterministic checker findings with +# the human-audit reference defects. This review measures checker coverage, +# not model performance. +uv run visionaid-evaluate review \ + --run-dir "$RUN_DIR" \ + --kind programmatic + +# Manually complete the generated programmatic Matches workbook. + +# 5. Validate and save the programmatic match decisions. +uv run visionaid-evaluate review \ + --run-dir "$RUN_DIR" \ + --kind programmatic \ + --import-matches + +# 6. Display the saved billable intent, request approval, and run the model. +# The command preserves raw responses, usage, failures, and canonical audit +# findings. It cannot overwrite evidence from an earlier live attempt. +uv run visionaid-evaluate audit \ + --live \ + --run-dir "$RUN_DIR" + +# 7. Generate a workbook that compares the model's findings with the eligible +# human-audit reference defects. These matches determine model recall. +uv run visionaid-evaluate review \ + --run-dir "$RUN_DIR" \ + --kind audit + +# Manually complete the generated audit Matches workbook. + +# 8. Validate and save the audit match decisions used for model scoring. +uv run visionaid-evaluate review \ + --run-dir "$RUN_DIR" \ + --kind audit \ + --import-matches + +# 9. Verify that the live evidence and both reviews belong to this run, then +# calculate the score and write JSON, CSV, and Markdown reports. +uv run visionaid-evaluate report --run-dir "$RUN_DIR" +``` + +## CLI workflow in detail + +The four commands describe stages of one evaluation rather than four unrelated +utilities. Each stage produces evidence that the next stage verifies and uses. + +### `prepare` + +- I added `prepare` to establish the benchmark before any model is evaluated. + It requires an authorized local workbook rather than downloading or + substituting private source data. + +- It creates the ignored `.model-evaluation` workspace so private inputs and + derived evidence stay outside version control. + +- When supplied with the approved workbook checksum and homepage URL, it + verifies the workbook, imports every applicable Home and Global row, and + preserves the original sheet, row, wording, location, WCAG evidence, and + recommendation. + +- It writes `references/imported.json` as the unapproved reference inventory + and `reviews/eligibility.xlsx` as the human-readable classification + workbook. Importing a row does not make it scorable until its eligibility + decision is reviewed and accepted. + +- When supplied with a snapshot URL, it captures the exact HTML that the audit + pipeline will consume and records the source URL, capture time, HTTP + metadata, temporal assumption, and SHA-256. + +- It refuses to overwrite or silently recapture an existing snapshot because + changing page content between model runs would invalidate the comparison. + +- `prepare` makes no provider request and incurs no model cost. + +### `audit` + +- I made `audit` dry by default because possessing an API key should never be + enough to spend money. + +- In planning mode, it requires an accepted reference set, the frozen + snapshot, and a positive maximum audit cost. It then runs the production + pipeline without provider calls to determine the exact prompts and payloads + that would be sent. + +- Planning creates a unique evaluation-run directory and freezes the approved + reference set, prompt text, payloads, prompt hashes, parser identity, model + configuration, pricing identity, retry policy, repository state, estimated + input usage, and cost guardrail. + +- It prints the model, endpoint, reasoning effort, benchmark identity, prompt + names, request count, retry policy, estimated input usage, maximum cost, and + output destination so billable intent is visible before execution. + +- Live mode is a separate invocation that requires `--live`, the planned run + directory, an API key, and interactive or explicit automatic approval. It + revalidates the saved evidence before creating the provider client. + +- Live execution sends prompts sequentially, records every attempt, applies + bounded retries only to transient failures, tracks billed usage against the + saved cost limit, and stops before another request could exceed that limit. + +- It preserves raw responses and provider metadata, then normalizes successful + responses into canonical audit findings. Authentication errors, malformed + output, exhausted retries, and incomplete runs remain recorded instead of + disappearing from the result. + +- A live destination is write-once. Repeating an experiment requires a newly + planned run so evidence from separate approvals cannot be mixed. + +### `review` + +- I added `review` because neither benchmark eligibility nor semantic finding + matches can be decided safely from string similarity alone. + +- The first review occurs after preparation. A reviewer classifies every + imported reference in `eligibility.xlsx`, records a rationale, decision, + identity, confidence, and timestamp, then imports the completed workbook as + `references/approved.json`. + +- The second review occurs per evaluation run. The command creates separate + match workbooks for `audit` findings and `programmatic` findings so model + performance cannot be confused with deterministic checker coverage. + +- Each match workbook presents reference evidence and finding evidence side by + side. The reviewer decides whether they describe the same underlying + accessibility failure at a compatible page, location, element, or element + group. + +- Importing a completed match workbook validates decision states, reviewer + provenance, evidence compatibility, required sub-defects, and one-to-one + matching before it writes the decision JSON used for scoring. + +- Candidate generation and workbook export never award points. The scorer + receives only accepted, validated decisions. + +### `report` + +- I added `report` as the final deterministic stage so presentation cannot + change the approved score. + +- It accepts an evaluation-run directory and discovers the frozen reference + set, canonical audit findings, canonical programmatic findings, both sets of + approved match decisions, and the live manifest. + +- It verifies that those artifacts belong to the same run and refuses to + report when live execution, normalization, or either review is missing. + +- It calculates workbook-row recall, programmatic coverage, combined workbook + coverage, estimated audit cost, tokens, latency, unmatched findings, and + format failures from the verified evidence. + +- It writes JSON, CSV, and Markdown from one score object so all three formats + agree on totals and individual workbook-row dispositions. + +- `report` does not call a model or reinterpret review decisions. Running it + again against unchanged evidence produces the same scoring result. + +## Shared model request layer + +- I created `AuditRequestClient` so model evaluations exercise the same + provider request code used in production. This prevents request behavior from + drifting between the application and the evaluation suite. + +- I added `AuditRequestConfig` to control and record the model, reasoning + effort, temperature, and maximum output tokens for each run. + +- I added configurable reasoning effort so OpenAI reasoning models can be + compared at a known setting. + +- I allowed temperature to be omitted because some models reject the parameter + rather than accepting a value. + +- I added model capability checks to prevent unsupported parameters from + causing failed requests. + +- I preserved production's existing temperature and output-token defaults so + moving production to the shared client would not change current audit + behavior. + +- I added automatic routing for Anthropic, OpenAI, and Gemini so callers do not + need provider-specific request logic. + +- I recorded requested and resolved configuration so each evaluation shows + what settings the provider received. + +- I added detailed token usage for cost calculations, including cached and + reasoning tokens when providers report them. + +- I added provider response details, stop reasons, duration, and structured + failures so incomplete runs can be diagnosed without repeating billable + requests. + +- I added provider-client injection so tests can inspect exact request + parameters without making live API calls. + +## Private inputs and benchmark evidence + +- I created an ignored `.model-evaluation` workspace because the source + workbook, derived references, captured HTML, reviews, raw responses, and + reports contain private project data. + +- I added workbook checksum validation because a changed or substituted + workbook would silently change the benchmark. + +- I preserved each reference defect's original workbook sheet, row, wording, + location, WCAG evidence, and recommendation so every score can be traced back + to its source. + +- I added an immutable homepage snapshot with its URL, capture time, HTTP + metadata, and SHA-256 because every model must receive the same page content. + +## Eligibility and scoring + +- I added eligibility review because the model should only lose points for + defects it had enough evidence and responsibility to detect. + +- I defined four eligibility classifications: `llm_eligible`, `programmatic`, + `unavailable_evidence`, and `ambiguous`. + +- `llm_eligible` means the audit model receives enough evidence in its prompt + and is responsible for making the required accessibility judgment. Only + accepted references in this class enter the model-recall denominator. This + classification does not assume the model will find the defect. + +- `programmatic` means the defect belongs to a deterministic checker rather + than the audit model. The row counts toward programmatic and combined + coverage only when an actual programmatic finding is reviewed and matched; + the classification alone does not award credit. + +- `unavailable_evidence` means the current audit pipeline does not receive the + visual, styling, interaction, cross-page, or other evidence needed to judge + the workbook claim. It identifies a benchmark limitation, not a model miss + and not a claim that the source defect is invalid. + +- `ambiguous` means the workbook claim, expected scope, or required sub-defects + cannot yet be interpreted confidently enough for fair assignment or + scoring. It describes uncertainty in the reference, not uncertainty in a + model response. + +- An accepted `ambiguous` classification is a completed review decision. It is + different from an eligibility decision that still has a `needs_review` + state. + +- I kept `unavailable_evidence` and `ambiguous` references visible in reports + but excluded them from both the model and programmatic denominators. This + prevents missing evidence or unresolved benchmark meaning from helping or + hurting a model. + +- I separated programmatic coverage from model recall because deterministic + checker results should not improve or reduce a model's ranking. + +- I added `EligibilityReviewer` and `MatchReviewer` interfaces so human + workbook review and a future Astra reviewer can produce the same validated + decisions without changing scoring or reporting. + +- I kept the scorer separate from review so it calculates results only from + approved decisions and produces the same result from the same inputs. + +- I made workbook-row recall over accepted `llm_eligible` defects the primary + model metric because the mandated workbook is the benchmark source. + +- I made estimated audit cost the only ranking tie-breaker. Token usage, + latency, unmatched findings, malformed output, and programmatic coverage + remain visible but do not alter rank. + +## Run safety and reproducibility + +- I added evaluation-run planning because the model configuration, prompt set, + benchmark, reference set, parser versions, pricing, retry policy, and cost + limit must be fixed before a billable request begins. + +- I added a positive cost guardrail, explicit `--live` flag, API-key + requirement, and point-of-use approval because credentials alone must never + trigger spending. + +- I made live-run output immutable because retrying into the same directory + would mix evidence from separate executions. + +- I added bounded retries for timeouts, rate limits, and server failures + because those failures may be temporary. + +- I disabled retries for authentication errors, invalid requests, and + malformed successful responses because repeating them would spend money + without addressing the cause. + +## Normalization, review, and reporting + +- I added canonical finding normalization for every registered non-summary + prompt because provider responses use different JSON shapes that cannot be + compared directly. + +- I preserved malformed responses and parsing failures instead of dropping + them because output-format reliability is part of the evaluation evidence. + +- I added candidate-match workbooks because semantic accessibility findings + cannot be matched safely by WCAG number or text similarity alone. + +- I enforced one-to-one approved matches because allowing one finding to + satisfy several unrelated workbook rows would inflate recall. + +- I added JSON, CSV, and Markdown reports from one score object so every format + agrees on totals, row dispositions, coverage, cost, and failures. + +## Verification + +- I added synthetic tests for the complete lifecycle so CI can verify the + suite without the private workbook, public network access, provider + credentials, or billable requests. + +- The current test suite passes all 76 tests. + +- A production pipeline dry run also passed without provider calls. It produced + 14 programmatic findings and generated 10 model prompts from the committed + HTML fixture. + +## Current proof-of-concept status + +- The evaluation framework is complete. Eleven implementation tickets are + resolved, and the remaining ticket covers the real Luna homepage proof of + concept. + +- The private homepage benchmark contains 16 reviewed reference defects: five + programmatic, three `llm_eligible`, six unavailable to the current pipeline, + and two ambiguous. + +- The three `llm_eligible` defects form the current model-recall denominator. + Unavailable and ambiguous defects remain reported without affecting the + score. + +- The frozen Pristine homepage snapshot and its provenance are saved in the + private workspace. + +- The benchmark dry run generated 29 programmatic findings and eight active + model prompts. + +- Five programmatic workbook rows have accepted one-to-one matches. The + remaining programmatic candidates were reviewed and rejected. + +- The first Luna run was planned for eight sequential requests with 5,776 + estimated input tokens, medium reasoning, no temperature parameter, and a + $1.00 cost limit. + +- The provider rejected the first live request because the API key was invalid. + The runner stopped immediately, used zero tokens, incurred no cost, and + marked the run incomplete and unrankable. + +- The failed run remains preserved rather than overwritten. + +- A fresh replacement run has been planned, and its five programmatic match + decisions have been validated. + +- The replacement run has not made a live provider request. It is waiting for + a valid OpenAI Platform API key. + +- There is no Luna efficacy score yet. Reporting a score before the replacement + live run and audit-match review would misrepresent the current state. + +## Remaining work + +- Replace the rejected credential with a valid OpenAI Platform API key. + +- Execute the prepared replacement Luna run. + +- Generate and review the audit-finding match workbook. + +- Import the approved audit match decisions. + +- Generate the final JSON, CSV, and Markdown evaluation reports. diff --git a/CONTEXT.md b/CONTEXT.md index ccfa6a4..1dbdc14 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -4,6 +4,10 @@ This context describes the language used to evaluate AI-generated accessibility ## Language +**Reference workbook**: +The authorized Excel workbook containing the human accessibility audit. Its worksheets provide source evidence for reference defects; generated eligibility and match workbooks are review artifacts, not the reference workbook. +_Avoid_: Ground-truth spreadsheet, review workbook, evaluation report + **Reference defect**: A human-reviewed accessibility problem derived from the mandated defect-report workbook and retained as benchmark evidence with traceable source provenance. _Avoid_: Ground-truth row, Excel issue, expected finding @@ -20,6 +24,18 @@ _Avoid_: Precision, F1, aggregate quality score A reference defect for which the AI audit model receives enough evidence and responsibility to make the relevant accessibility judgment. Defects intentionally handled only by programmatic checks are not LLM-eligible. _Avoid_: Every workbook row, programmatic defect +**Programmatic reference defect**: +A reference defect assigned to deterministic audit checks rather than the AI audit model. Classification alone does not award coverage; an approved programmatic match is still required. +_Avoid_: Programmatic finding, automatically covered defect + +**Unavailable-evidence reference defect**: +A reference defect that the current audit pipeline cannot judge because its inputs omit required visual, styling, interaction, cross-page, or other evidence. +_Avoid_: Model miss, invalid reference defect + +**Ambiguous reference defect**: +A reference defect whose claim, scope, or required sub-defects cannot yet be interpreted confidently enough for fair responsibility assignment or scoring. An accepted ambiguous classification is a completed review decision, not an unresolved review state. +_Avoid_: Model uncertainty, unavailable-evidence reference defect + **Benchmark snapshot**: An immutable capture of the audited webpage used as the common input for every evaluated model, with its source and capture identity recorded. _Avoid_: Live page, test URL From 617a31f8e69fab475c64dbe7f94658b392617743 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 20 Sep 2026 13:40:47 -0400 Subject: [PATCH 026/126] feat(evaluation): removed temporal assumption metadata from evaluation --- .../03-freeze-pristine-benchmark-snapshot.md | 2 +- .scratch/model-evaluation/progress-report.md | 5 ++-- .scratch/model-evaluation/spec.md | 3 +-- README.md | 13 ++++------ tests/test_evaluation_cli.py | 19 --------------- vision_aid/evaluation/cli.py | 12 ---------- vision_aid/evaluation/snapshot.py | 24 ------------------- 7 files changed, 9 insertions(+), 69 deletions(-) diff --git a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md index 46a5caf..5f83e68 100644 --- a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md +++ b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md @@ -7,7 +7,7 @@ **Status:** resolved - [x] The preparation workflow saves the exact raw HTML consumed by the existing audit pipeline. -- [x] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, SHA-256, and the stakeholder-supported temporal assumption. +- [x] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, and SHA-256. - [x] Reusing a snapshot does not refetch or silently overwrite it. - [x] The existing unrelated DAT Vision Aid fixture is rejected as a Pristine benchmark input. - [x] Tests use local HTTP fakes and verify content identity, provenance, hashing, and overwrite protection. diff --git a/.scratch/model-evaluation/progress-report.md b/.scratch/model-evaluation/progress-report.md index 85576ee..73410bb 100644 --- a/.scratch/model-evaluation/progress-report.md +++ b/.scratch/model-evaluation/progress-report.md @@ -54,8 +54,7 @@ command 3. uv run visionaid-evaluate prepare \ --workbook-sha256 "$WORKBOOK_SHA256" \ --homepage-url https://pristineai.com/ \ - --snapshot-url https://pristineai.com/ \ - --temporal-assumption "$TEMPORAL_ASSUMPTION" + --snapshot-url https://pristineai.com/ # Manually complete .model-evaluation/reviews/eligibility.xlsx. # Classify who is responsible for each reference defect before scoring begins. @@ -139,7 +138,7 @@ utilities. Each stage produces evidence that the next stage verifies and uses. - When supplied with a snapshot URL, it captures the exact HTML that the audit pipeline will consume and records the source URL, capture time, HTTP - metadata, temporal assumption, and SHA-256. + metadata, and SHA-256. - It refuses to overwrite or silently recapture an existing snapshot because changing page content between model runs would invalidate the comparison. diff --git a/.scratch/model-evaluation/spec.md b/.scratch/model-evaluation/spec.md index 62c1224..09452f7 100644 --- a/.scratch/model-evaluation/spec.md +++ b/.scratch/model-evaluation/spec.md @@ -40,7 +40,6 @@ Private inputs and generated artifacts will stay outside version control. The co 22. As a data owner, I want future Astra review to use the authorized API project without publishing inputs, so that automation does not weaken data privacy. 23. As an evaluator, I want a benchmark snapshot captured once from the Pristine homepage, so that every model sees identical page content. 24. As an evaluator, I want the snapshot URL, capture time, HTTP metadata, and SHA-256 recorded, so that the tested page state is identifiable. -25. As a project lead, I want the stakeholder assertion that Pristine has not changed since the workbook audit recorded with the snapshot, so that the temporal assumption is explicit. 26. As an evaluator, I want Global rows scored only for their homepage manifestation in this proof of concept, so that homepage evidence is not inflated into a site-wide claim. 27. As an evaluator, I want the existing programmatic audit run against the benchmark snapshot, so that deterministic workbook coverage and checker gaps are measured. 28. As an evaluator, I want programmatic findings excluded from the AI model score, so that model-independent results do not inflate similarity between models. @@ -101,7 +100,7 @@ Private inputs and generated artifacts will stay outside version control. The co - The private source workbook remains at the repository root under its current name by default and is ignored by exact filename. A command-line override permits another local location. The importer validates a recorded SHA-256 and fails clearly on absence or drift. - All private derived artifacts live under one ignored evaluation workspace partitioned into references, reviews, snapshots, runs, and reports. Only code, schemas, documentation, synthetic fixtures, and public pricing data are committed. - `openpyxl` is added as a runtime dependency for workbook import and private review workbooks. `pytest` is added as a development dependency. -- The Pristine homepage is captured as raw HTML because that is the production pipeline's input. The capture is immutable and accompanied by source URL, retrieval time, HTTP metadata, SHA-256, and a note that stakeholders report no change since the workbook audit. +- The Pristine homepage is captured as raw HTML because that is the production pipeline's input. The capture is immutable and accompanied by source URL, retrieval time, HTTP metadata, and SHA-256. - The existing DAT Vision Aid HTML fixture is unrelated to the workbook and is never used for workbook-based scoring. - The proof of concept includes workbook rows scoped as Home and Global on the Pristine homepage. Global rows are labeled as global claims supported by homepage evidence and are not multiplied across pages. - Every source row becomes one reviewed reference record. Raw text and provenance are retained. Normalized fields include the canonical problem, location or element group, WCAG evidence, eligibility classification, rationale, approval state, and any required sub-defects. diff --git a/README.md b/README.md index 5c3436c..a481760 100644 --- a/README.md +++ b/README.md @@ -249,22 +249,19 @@ uv run visionaid-evaluate prepare ``` To freeze the Pristine homepage for model comparisons, supply its authorized -URL and record the stakeholder-supported temporal assumption during preparation: +URL during preparation: ```bash uv run visionaid-evaluate prepare \ - --snapshot-url "$PRISTINE_HOMEPAGE_URL" \ - --temporal-assumption \ - "Stakeholders report that the homepage has not changed since the workbook audit." + --snapshot-url "$PRISTINE_HOMEPAGE_URL" ``` The command saves the HTTP response body without HTML transformation as `.model-evaluation/snapshots/pristine-homepage/source.html`. The adjacent `metadata.json` records the source and final URLs, retrieval time, HTTP status, -request user agent, response identity headers, byte length, SHA-256, and -temporal assumption. Both files are staged and published as one immutable -bundle. Repeating preparation for the same URL verifies and reuses that bundle -without another request. +request user agent, response identity headers, byte length, and SHA-256. Both +files are staged and published as one immutable bundle. Repeating preparation +for the same URL verifies and reuses that bundle without another request. Checksum drift, changed provenance, or a partial snapshot fails closed; prepare never silently replaces an existing benchmark. Local HTML files, including the unrelated DAT Vision Aid fixture, cannot be used as the Pristine snapshot. diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index adc8c0b..9fd2747 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -30,10 +30,6 @@ SNAPSHOT_SHA256 = ( "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" ) -TEMPORAL_ASSUMPTION = ( - "Stakeholders report that the homepage has not changed since the " - "workbook audit." -) class SnapshotHTTPServer(ThreadingHTTPServer): @@ -293,8 +289,6 @@ def test_prepare_captures_exact_html_with_snapshot_provenance( str(synthetic_workbook), "--snapshot-url", source_url, - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) @@ -314,7 +308,6 @@ def test_prepare_captures_exact_html_with_snapshot_provenance( assert metadata["source_url"] == source_url assert metadata["sha256"] == SNAPSHOT_SHA256 assert metadata["byte_length"] == len(SNAPSHOT_BODY) - assert metadata["temporal_assumption"] == TEMPORAL_ASSUMPTION assert metadata["html_file"] == html_path.name assert metadata["http"] == { "content_length": str(len(SNAPSHOT_BODY)), @@ -341,8 +334,6 @@ def test_prepare_reuses_a_verified_snapshot_without_refetching( str(synthetic_workbook), "--snapshot-url", source_url, - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) second = run_cli( @@ -372,8 +363,6 @@ def test_prepare_rejects_html_the_audit_pipeline_cannot_decode( str(synthetic_workbook), "--snapshot-url", source_url.replace("/pristine-homepage", "/non-utf8"), - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) @@ -395,8 +384,6 @@ def test_prepare_refuses_to_overwrite_a_changed_snapshot( str(synthetic_workbook), "--snapshot-url", source_url, - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) assert first.returncode == 0, first.stderr @@ -452,8 +439,6 @@ def publish_competing_snapshot() -> None: str(synthetic_workbook), "--snapshot-url", source_url, - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) @@ -475,8 +460,6 @@ def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( str(synthetic_workbook), "--snapshot-url", str(PROJECT_ROOT / "test_files" / "dat_visionaid_home.html"), - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) @@ -717,8 +700,6 @@ def test_cli_completes_the_synthetic_private_workflow( "https://example.test/", "--snapshot-url", source_url, - "--temporal-assumption", - TEMPORAL_ASSUMPTION, cwd=tmp_path, ) assert prepared.returncode == 0, prepared.stderr diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 341b481..7a2a6df 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -112,13 +112,6 @@ def build_parser() -> argparse.ArgumentParser: "--snapshot-url", help="HTTP(S) source URL for an immutable Pristine homepage capture", ) - prepare.add_argument( - "--temporal-assumption", - help=( - "stakeholder-supported statement relating a new capture to the " - "workbook audit" - ), - ) prepare.set_defaults(handler=_prepare) audit = workflows.add_parser( @@ -223,15 +216,10 @@ def _prepare(arguments: argparse.Namespace) -> int: export_eligibility_workbook(reference_set, review_path) print(f"Imported reference set: {imported_path.resolve()}") print(f"Eligibility review workbook: {review_path.resolve()}") - if arguments.temporal_assumption and not arguments.snapshot_url: - raise PrivateInputError( - "--temporal-assumption requires --snapshot-url." - ) if arguments.snapshot_url: snapshot = prepare_benchmark_snapshot( workspace, source_url=arguments.snapshot_url, - temporal_assumption=arguments.temporal_assumption, ) action = "reused" if snapshot.reused else "captured" print(f"Benchmark snapshot {action}: {snapshot.html_path.resolve()}") diff --git a/vision_aid/evaluation/snapshot.py b/vision_aid/evaluation/snapshot.py index a896de5..3e1686d 100644 --- a/vision_aid/evaluation/snapshot.py +++ b/vision_aid/evaluation/snapshot.py @@ -42,7 +42,6 @@ class SnapshotMetadata: retrieved_at: str sha256: str source_url: str - temporal_assumption: str def to_json(self) -> object: """Return the JSON-compatible representation stored beside the HTML.""" @@ -62,7 +61,6 @@ def from_json(cls, value: object) -> "SnapshotMetadata": "retrieved_at", "sha256", "source_url", - "temporal_assumption", ) if any(not isinstance(value.get(name), str) for name in required_strings): raise ValueError("snapshot metadata has a missing string field") @@ -109,7 +107,6 @@ def from_json(cls, value: object) -> "SnapshotMetadata": retrieved_at=cast(str, value["retrieved_at"]), sha256=cast(str, value["sha256"]), source_url=cast(str, value["source_url"]), - temporal_assumption=cast(str, value["temporal_assumption"]), ) @@ -126,7 +123,6 @@ class BenchmarkSnapshot: def prepare_benchmark_snapshot( workspace: PrivateWorkspace, source_url: str, - temporal_assumption: str | None, ) -> BenchmarkSnapshot: """Capture a benchmark once or verify an existing immutable capture.""" _require_http_source(source_url) @@ -141,14 +137,6 @@ def prepare_benchmark_snapshot( html_path, metadata_path, source_url, - temporal_assumption, - ) - - assumption = (temporal_assumption or "").strip() - if not assumption: - raise PrivateInputError( - "A new benchmark snapshot requires --temporal-assumption so the " - "stakeholder-supported relationship to the workbook audit is explicit." ) import requests @@ -203,7 +191,6 @@ def prepare_benchmark_snapshot( retrieved_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"), sha256=hashlib.sha256(body).hexdigest(), source_url=source_url, - temporal_assumption=assumption, ) _save_new_snapshot(snapshot_directory, body, metadata) return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=False) @@ -229,7 +216,6 @@ def _load_snapshot( html_path: Path, metadata_path: Path, source_url: str, - temporal_assumption: str | None, ) -> BenchmarkSnapshot: """Validate an existing snapshot without fetching or changing it.""" if not html_path.is_file() or not metadata_path.is_file(): @@ -252,16 +238,6 @@ def _load_snapshot( f"{metadata.source_url!r}, " f"not {source_url!r}. It will not be refetched or overwritten." ) - supplied_assumption = (temporal_assumption or "").strip() - if ( - supplied_assumption - and supplied_assumption != metadata.temporal_assumption - ): - raise PrivateInputError( - "The supplied temporal assumption differs from the immutable snapshot " - "metadata. The existing snapshot will not be overwritten." - ) - try: body = html_path.read_bytes() except OSError as error: From 96c5a4fefe335fef8965c9b2aba0866acfd6c027 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 22 Sep 2026 14:24:40 -0400 Subject: [PATCH 027/126] feat(evaluation): retarget frozen audit to Haiku extended thinking --- .../anthropic-evaluation-retarget/spec.md | 129 ++++++++++++++++ .scratch/evaluation-cli-workflow/spec.md | 7 + ...01-explicit-audit-request-configuration.md | 7 + .../issues/07-safe-luna-dry-run.md | 7 + .../issues/08-authorized-live-audit.md | 7 + .../issues/12-run-luna-homepage-poc.md | 17 ++- .scratch/model-evaluation/progress-report.md | 9 ++ .scratch/model-evaluation/spec.md | 7 + README.md | 18 ++- processing_scripts/llm_client/audit.py | 124 +++++++++++----- pyproject.toml | 2 +- tests/test_audit_request.py | 93 ++++++++++-- tests/test_evaluation_audit.py | 58 ++++---- tests/test_evaluation_cli.py | 140 +++++++++++------- tests/test_scoring_and_reporting.py | 14 +- vision_aid/evaluation/audit.py | 27 ++-- vision_aid/evaluation/cli.py | 13 +- vision_aid/evaluation/pricing.py | 12 +- vision_aid/evaluation/pricing.v2.json | 24 +++ vision_aid/evaluation/reporting.py | 6 +- vision_aid/evaluation/schemas.py | 2 +- vision_aid/evaluation/scoring.py | 2 +- vision_aid/evaluation/serialization.py | 28 ++-- 23 files changed, 562 insertions(+), 191 deletions(-) create mode 100644 .scratch/anthropic-evaluation-retarget/spec.md create mode 100644 vision_aid/evaluation/pricing.v2.json diff --git a/.scratch/anthropic-evaluation-retarget/spec.md b/.scratch/anthropic-evaluation-retarget/spec.md new file mode 100644 index 0000000..1032d56 --- /dev/null +++ b/.scratch/anthropic-evaluation-retarget/spec.md @@ -0,0 +1,129 @@ +Status: ready-for-agent + +## Problem Statement + +The private accessibility-audit evaluation workflow assumes that its API credential is an OpenAI key and freezes GPT-5.6 Luna at medium reasoning into every planned evaluation run. The available credential is instead an Anthropic key. As a result, the evaluator cannot execute the intended proof of concept even though the repository's shared audit path already supports Anthropic and the public application already defaults to Claude Haiku 4.5. + +The evaluator must be retargeted without turning this focused proof of concept into a general provider-selection system. The replacement must preserve the existing benchmark snapshot, audit prompts, review workflow, deterministic scoring, workbook-row recall ranking, cost reporting, write-once evaluation artifacts, and live-run approval protections. Historical Luna evaluation runs must remain immutable and readable. + +## Solution + +Retarget the private evaluation workflow from GPT-5.6 Luna to the pinned Anthropic model `claude-haiku-4-5-20251001`. Configure every audit request with manual extended thinking capped at 16,000 tokens and a total output cap of 24,192 tokens, leaving up to the existing 8,192-token allowance for the final audit JSON when the full thinking budget is used. Omit temperature and use Anthropic's Messages API. + +Extend the existing shared audit-client boundary only as much as required to send this fixed thinking configuration and consume the streaming response required by the larger output cap. Return the same normalized final-text and usage contract that the evaluator already consumes. Thinking and redacted-thinking blocks will not become evaluation evidence or be persisted; aggregate provider usage will still be retained and priced. + +The evaluation CLI will resolve `ANTHROPIC_API_KEY`, display the exact pinned model, thinking budget, total output cap, endpoint, pricing identity, and cost guardrail in its plan and approval summaries, and continue to be dry by default. The implementation will generate a fresh dry-run plan under a new evaluation run identity. It will not execute a paid live audit. + +## User Stories + +1. As an evaluation operator, I want the private evaluator to use my Anthropic credential, so that I can continue the proof of concept with the credential I was actually issued. +2. As an evaluation operator, I want a missing credential error to name `ANTHROPIC_API_KEY`, so that setup instructions do not send me toward the wrong provider. +3. As an evaluator, I want the audit model pinned to `claude-haiku-4-5-20251001`, so that a moving provider alias cannot silently change an evaluation run. +4. As an evaluator, I want the provider recorded as Anthropic, so that the run manifest accurately identifies the service that generated each audit finding. +5. As an evaluator, I want the endpoint recorded as Anthropic Messages, so that the request protocol is reproducible. +6. As an evaluator, I want Haiku extended thinking enabled, so that the low-cost model has additional reasoning capacity for accessibility judgments. +7. As an evaluator, I want the thinking budget fixed at 16,000 tokens, so that every request in the evaluation run has the same reproducible ceiling. +8. As an evaluator, I want the total output cap fixed at 24,192 tokens, so that a request can use its thinking budget without reducing the prior 8,192-token final-response allowance. +9. As an evaluator, I want temperature omitted while thinking is enabled, so that the request satisfies Anthropic's supported contract. +10. As an evaluation operator, I want the exact thinking and output ceilings shown before execution, so that the potential billed usage is visible. +11. As a budget owner, I want Haiku input and output usage priced from a versioned public price schedule, so that estimated run cost remains reproducible. +12. As a budget owner, I want all billed output tokens included in estimated run cost, including tokens used for thinking, so that cost is not understated. +13. As a budget owner, I want aggregate Anthropic usage preserved without inventing a reasoning-token breakdown that the response does not supply, so that reports distinguish measured values from unavailable detail. +14. As an evaluation operator, I want streaming handled internally by the audit client, so that I receive the same final audit result contract as before. +15. As an evaluator, I want streamed text blocks assembled in provider order, so that the final audit JSON can be normalized correctly. +16. As an evaluator, I want provider stop reasons and aggregate usage retained after streaming, so that format failures, truncation, and cost remain reviewable. +17. As a data owner, I want thinking and redacted-thinking blocks discarded rather than stored, so that internal reasoning does not become benchmark evidence or a new private artifact class. +18. As a comparison reviewer, I want only the model's final audit text normalized into canonical audit findings, so that the review contract remains unchanged. +19. As a comparison reviewer, I want malformed or missing final text handled by the existing normalization policy, so that thinking support does not conceal structured-output failure. +20. As an evaluator, I want the benchmark snapshot, prompts, filters, prompt order, and disabled summaries preserved, so that the provider retarget is the material experimental change. +21. As an evaluator, I want workbook-row recall to remain the primary ranking metric, so that changing providers does not change the definition of model performance. +22. As an evaluator, I want estimated run cost to remain only the tie-breaker, so that the higher per-token price of Haiku does not silently redefine model quality. +23. As an operator, I want the evaluation to remain dry by default even when `ANTHROPIC_API_KEY` exists, so that environment configuration alone cannot spend money. +24. As an operator, I want live execution to retain its explicit flag, positive cost guardrail, and point-of-use live-run approval, so that the provider change cannot bypass spending controls. +25. As an operator, I want this change to create a fresh evaluation run plan, so that its provider, model, thinking, output, and pricing identities are internally consistent. +26. As an operator, I want the fresh plan to remain unexecuted, so that I can review it before authorizing a paid run. +27. As an evaluator, I want the earlier failed Luna run preserved, so that the historical authentication failure remains auditable. +28. As an evaluator, I want the prepared but unexecuted Luna replacement plan preserved and marked as superseded or abandoned, so that immutable run history is not rewritten. +29. As an evaluator, I want old Luna artifacts to remain loadable for reporting and review where supported, so that current configuration changes do not corrupt historical evidence. +30. As an evaluator, I want an old Luna plan rejected for new live execution after the retarget, so that a stale plan cannot run under Haiku credentials or pricing. +31. As a maintainer, I want the public web application and HTTP API left unchanged, so that an evaluator-only retarget does not disturb existing multi-provider behavior. +32. As a maintainer, I want the existing audit-client interface extended rather than duplicated, so that Anthropic request and usage handling remain centralized. +33. As a maintainer, I want thinking support represented explicitly rather than disguised as OpenAI `reasoning_effort`, so that provider-specific semantics remain truthful. +34. As a maintainer, I want streaming enabled only where the configured Anthropic request requires it, so that unrelated audit calls retain their current behavior. +35. As a maintainer, I want existing non-thinking Anthropic calls to keep working, so that the focused evaluator change does not regress production audits. +36. As a maintainer, I want existing OpenAI and Gemini client behavior to remain unchanged, so that this retarget has a narrow regression surface. +37. As a maintainer, I want tests to use fake provider responses, so that verification never requires a credential, network access, or paid tokens. +38. As a maintainer, I want documentation to identify Haiku as the current proof-of-concept target, so that operators do not follow obsolete Luna instructions. +39. As a maintainer, I want historical Luna implementation records retained with supersession notes rather than rewritten, so that the issue tracker remains truthful. +40. As a future evaluator, I want the model identifier and pricing isolated in the existing frozen configuration and price schedule, so that a later retirement can be handled without redesigning the framework. + +## Implementation Decisions + +- This is a fixed-target retarget of the private evaluation workflow, not a general provider-selection feature. +- The pinned model is `claude-haiku-4-5-20251001`; the moving `claude-haiku-4-5` alias will not be used for an evaluation run. +- The request protocol is Anthropic Messages and the credential environment variable is `ANTHROPIC_API_KEY`. +- Manual extended thinking is enabled with `budget_tokens` set to 16,000. Haiku does not support adaptive thinking or an OpenAI-style `medium` reasoning setting. +- The request's `max_tokens` value is 24,192. Thinking and final text share this provider limit; the selected value preserves up to 8,192 tokens for final text when the full thinking budget is consumed. +- Temperature, `top_k`, and `top_p` are omitted from thinking-enabled requests. +- Thinking configuration is modeled explicitly in the shared audit request configuration. It will not overload the OpenAI-specific `reasoning_effort` field. +- The shared audit client gains the minimum streaming behavior required for this Anthropic configuration. It assembles final text, captures the terminal stop reason and usage, and returns the existing provider-neutral audit result. +- Streaming is not introduced as a new public evaluation abstraction. It remains an internal transport detail selected by the Anthropic request requirements. +- Only final text blocks enter normalization. Thinking, signatures, and redacted-thinking blocks are neither returned as audit text nor persisted as raw evaluation evidence. +- Aggregate provider output usage is authoritative for billing. The implementation will not estimate or fabricate a separate Anthropic reasoning-token count. +- Existing usage and reporting schemas will be changed only where needed to avoid presenting unavailable Anthropic reasoning detail as a measured zero. +- Haiku pricing is added to the versioned price schedule using the official direct-API input and output rates current when the schedule is updated. Pricing source and effective date remain recorded. +- Prompt caching and Batch API discounts are not enabled. Their prices may be represented only if the existing schedule requires complete model metadata. +- The frozen evaluation configuration, pricing identity, and resulting benchmark identity change. A new evaluation run is therefore mandatory. +- Previous run directories are immutable. The failed Luna run and the unexecuted Luna replacement plan will not be edited, deleted, or repurposed. +- The approved reference set and benchmark snapshot may be reused because the provider retarget does not change their content. +- Programmatic match decisions may be carried into the fresh run only through the existing row-for-row semantic verification process because run-scoped identifiers change. +- The operator log will append the model-selection rationale, final request configuration, superseded Luna plan status, and fresh dry-run identifier. Historical entries will remain intact. +- Normative evaluation documentation and the currently open operational ticket will be updated for Haiku. Resolved Luna tickets and superseded specifications will receive historical clarification rather than retroactive rewriting. +- Public application behavior is unchanged because it already supports Anthropic and already defaults to the pinned Haiku model. +- The implementation ends after generating and verifying a fresh dry-run plan. It must not make a live Anthropic request. +- No new domain glossary term or ADR is required. The existing concepts of evaluation run, estimated run cost, live-run approval, audit finding, and benchmark snapshot cover the change, while the fixed model target remains a reversible specification decision. + +## Testing Decisions + +- Tests will assert observable request, result, plan, and safety behavior rather than private helper structure. +- Two existing high-level seams are sufficient: the shared audit client's public request/result boundary and the evaluation CLI's dry-run plan boundary. No new test-only seam will be introduced. +- At the audit-client seam, a fake Anthropic stream will verify the pinned model, Messages request shape, 16,000-token thinking budget, 24,192-token total cap, omission of sampling parameters, ordered final-text assembly, terminal stop reason, and aggregate usage. +- At the same seam, mixed thinking and text blocks will verify that only final text is exposed as audit output, while thinking-only or truncated responses produce the existing empty or malformed final-output behavior. +- Existing plain Anthropic tests will remain and prove that non-thinking requests do not accidentally acquire thinking or streaming behavior. +- Existing OpenAI and Gemini request-shape tests will remain unchanged and serve as regression coverage for the provider-specific branch. +- At the evaluation CLI seam, a dry-run test will verify that the manifest and human-readable summary contain the Anthropic provider, pinned model, Messages endpoint, enabled thinking budget, total output cap, omitted temperature, and new pricing identity. +- CLI tests will verify that `ANTHROPIC_API_KEY` is required only for live execution, that an OpenAI-named key is not accepted as the required Anthropic credential, and that dry-run planning requires no credential. +- Safety tests will verify that the explicit live flag, positive cost guardrail, and live-run approval remain mandatory and that no fake or real provider call occurs during the requested dry run. +- Historical-plan tests will verify that an old Luna plan remains readable where historical loading permits but is rejected for execution under the new frozen configuration. +- Pricing tests will calculate Haiku input and aggregate output costs from fixed synthetic usage, including output usage attributable to thinking, without depending on live provider pricing. +- Reporting tests will verify that unavailable provider-specific reasoning detail is not mislabeled as an API-reported zero while total billed output remains correct. +- Normalization and scoring tests should remain provider-neutral. Existing tests for canonical audit findings, malformed responses, deterministic workbook-row recall, and cost-only tie-breaking are prior art and should pass without behavioral changes. +- The complete focused regression suite will cover audit requests, evaluation audit planning/execution policy, evaluation CLI behavior, pricing, reporting, and normalization. No test may access the internet or spend money. +- A good acceptance test demonstrates the complete dry-run behavior through the CLI with fake provider boundaries: it produces a fresh Haiku plan with the exact frozen configuration and no network call. + +## Out of Scope + +- Supporting runtime selection between OpenAI, Anthropic, Gemini, or arbitrary providers in the private evaluator. +- Changing the public website, public HTTP API, or their existing Haiku default. +- Claiming that Claude Haiku 4.5 is officially equivalent to GPT-5.6 Luna at medium reasoning. +- Mapping Anthropic thinking budgets to OpenAI reasoning-effort labels. +- Adaptive thinking, dynamic thinking budgets, per-prompt budgets, retries at a stronger model, or automatic provider fallback. +- Generalizing streaming as a user-selectable mode or changing unrelated non-streaming calls. +- Persisting thinking summaries, encrypted thinking signatures, redacted-thinking blocks, or raw chain-of-thought. +- Changing audit prompts, filters, prompt order, summary behavior, canonical audit-finding normalization, matching rules, or deterministic scoring. +- Enabling prompt caching, Batch API execution, tool use, browser tools, code execution, or computer use. +- Optimizing estimated run cost beyond accurately pricing reported usage. +- Deleting, mutating, or reusing prior Luna evaluation runs. +- Reclassifying reference defects or changing approved match decisions except for the established run-ID carry-forward procedure. +- Executing the paid Haiku audit or granting live-run approval. +- Adding a new domain glossary term or architectural decision record for the provider retarget. + +## Further Notes + +- Claude Haiku 4.5 is the closest current Anthropic cost-first analogue to GPT-5.6 Luna, but this is a product-positioning inference rather than an official cross-provider equivalence. +- At the time of specification, direct Claude API pricing is $1 per million input tokens and $5 per million output tokens. The implementation must verify and record the official price source when updating the versioned schedule. +- Haiku's thinking budget is a ceiling and the model may stop thinking earlier. The provider bills thinking within aggregate output usage. +- The larger total output cap requires streaming under the current Anthropic SDK guidance. This is why a small shared-client change is necessary even though ordinary Anthropic Messages requests are already supported. +- The existing prepared Luna replacement plan cannot be converted in place because model, provider, endpoint, reasoning configuration, and pricing participate in immutable run identity and point-of-use verification. +- Model lifecycle should be checked again before a future paid run. The pinned identifier is active at specification time, but a dated model can eventually be deprecated or retired. +- This spec supersedes only the Luna-specific target and execution details of the broader model-evaluation proof of concept. Its domain model, benchmark methodology, review separation, scoring rules, and privacy boundaries remain authoritative. diff --git a/.scratch/evaluation-cli-workflow/spec.md b/.scratch/evaluation-cli-workflow/spec.md index 52a78c2..3cd2f34 100644 --- a/.scratch/evaluation-cli-workflow/spec.md +++ b/.scratch/evaluation-cli-workflow/spec.md @@ -124,3 +124,10 @@ Implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by the synthetic CLI lifecycle, focused approval and identity-drift tests, lower-level retry and cost tests, compilation, and the full network-free pytest suite. The operational Luna proof of concept remains open and no provider request was made. + + +## Haiku retarget clarification (2026-09-22) + +The Luna-specific target and credential details above are historical. They are +superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, +review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md index a069642..90094d8 100644 --- a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md +++ b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md @@ -16,3 +16,10 @@ - Implemented in `c371979` with review fixes in `c395efa`. Verified by four network-free audit-request tests and the production pipeline dry-run smoke test. + + +## Haiku retarget clarification (2026-09-22) + +The Luna-specific target and credential details above are historical. They are +superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, +review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md index a9b6dda..29de7a3 100644 --- a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md +++ b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md @@ -21,3 +21,10 @@ and identity-verified in `2129616`. The run-oriented planning workflow was implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by unique run, frozen-evidence, positive-cost, identity-drift, and no-spend tests; no provider call is made by the test suite. + + +## Haiku retarget clarification (2026-09-22) + +The Luna-specific target and credential details above are historical. They are +superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, +review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/08-authorized-live-audit.md b/.scratch/model-evaluation/issues/08-authorized-live-audit.md index 955e775..22800a6 100644 --- a/.scratch/model-evaluation/issues/08-authorized-live-audit.md +++ b/.scratch/model-evaluation/issues/08-authorized-live-audit.md @@ -24,3 +24,10 @@ and identity-verified in `2129616`. Point-of-use approval and write-once run execution were implemented in `2448042` and copy/rerun isolation was hardened in `6a1e6c0`. Fake-provider tests verify approval, retry, checkpoint, budget, malformed-response, and incomplete-run policies without spending. + + +## Haiku retarget clarification (2026-09-22) + +The Luna-specific target and credential details above are historical. They are +superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, +review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md index f7cae14..1a5d7cc 100644 --- a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md +++ b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md @@ -1,18 +1,18 @@ -# 12: Run the gated Luna homepage proof of concept +# 12: Run the gated Haiku homepage proof of concept -**What to build:** Execute the approved operational workflow on the private Pristine benchmark and produce the first illustrative Luna evaluation report without bypassing any human or spending gate. +**What to build:** Execute the approved operational workflow on the private Pristine benchmark and produce the first illustrative Haiku evaluation report without bypassing any human or spending gate. **Blocked by:** 11: Verify the complete evaluation workflow. **Status:** ready-for-agent - [ ] The private workbook checksum, snapshot, reference set, and eligibility classifications are reviewed and approved before the audit proceeds. -- [ ] A no-cost evaluation-run summary presents the exact Luna configuration, request set, estimated usage, and planned cost limit. +- [ ] A no-cost evaluation-run summary presents the exact Haiku configuration, request set, estimated usage, and planned cost limit. - [ ] The live run begins only after point-of-use approval covering the displayed configuration and saved cost guardrail. -- [ ] The approved Luna run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. +- [ ] The approved Haiku run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. - [ ] Canonical findings and candidate matches are generated without production CSV filtering or model-based deduplication. - [ ] Ambiguous match decisions receive human approval before scoring. -- [ ] The final reports show Luna workbook-row recall, programmatic coverage, combined workbook coverage, estimated audit cost, tokens, latency, unmatched findings, and format failures. +- [ ] The final reports show Haiku workbook-row recall, programmatic coverage, combined workbook coverage, estimated audit cost, tokens, latency, unmatched findings, and format failures. - [ ] The result is labeled illustrative and makes no small-versus-large model claim from the homepage's limited eligible denominator. ## Comments @@ -20,3 +20,10 @@ The run-oriented CLI ergonomics were implemented in `2448042` and hardened in `6a1e6c0`. This operational ticket remains open: no private preparation, live Luna request, human match review, or final proof-of-concept report was performed. + + +2026-09-22: The active target is now pinned `claude-haiku-4-5-20251001`, +with 16,000 thinking tokens and 24,192 total output tokens via Anthropic +Messages. See `../../anthropic-evaluation-retarget/spec.md`. This operational +ticket remains open: the retarget ends at a dry-run plan, with no live approval +or paid request. Historical Luna attempts remain immutable. diff --git a/.scratch/model-evaluation/progress-report.md b/.scratch/model-evaluation/progress-report.md index 73410bb..3c4376c 100644 --- a/.scratch/model-evaluation/progress-report.md +++ b/.scratch/model-evaluation/progress-report.md @@ -439,3 +439,12 @@ utilities. Each stage produces evidence that the next stage verifies and uses. - Import the approved audit match decisions. - Generate the final JSON, CSV, and Markdown evaluation reports. + + +## Haiku retarget (2026-09-22) + +The Luna replacement plan is superseded by a fresh, unexecuted Haiku plan. +Current configuration: Anthropic Messages, `claude-haiku-4-5-20251001`, +16,000 thinking tokens, 24,192 total output tokens, no temperature. See the +private operator log for the new run identity. No paid run is authorized or +performed by this retarget; operational ticket 12 remains open. diff --git a/.scratch/model-evaluation/spec.md b/.scratch/model-evaluation/spec.md index 09452f7..ab14ea3 100644 --- a/.scratch/model-evaluation/spec.md +++ b/.scratch/model-evaluation/spec.md @@ -175,3 +175,10 @@ Private inputs and generated artifacts will stay outside version control. The co - The workbook is the mandatory provenance source, but approved normalization may mark rows unavailable or ambiguous and may represent distinct required sub-defects. These transformations must remain traceable and reviewable. - The first implementation milestone ends at the eligibility-review gate and incurs no API cost. The cost limit for the first live Luna run will be selected while planning the evaluation run, whose summary exposes the request set and estimated usage. - Future benchmark expansion should sample distinct pages and template families rather than random rows because repeated job-page patterns dominate the workbook. + + +## Haiku retarget clarification (2026-09-22) + +The Luna-specific target and credential details above are historical. They are +superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, +review, scoring, and safety requirements remain authoritative. diff --git a/README.md b/README.md index a481760..f13c1ea 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ Only the four classifications `llm_eligible`, `programmatic`, `unavailable_evidence`, and `ambiguous` are accepted. Global rows remain one workbook-row scoring unit supported by homepage evidence. -`audit` is network-free by default even if `OPENAI_API_KEY` exists. With an +`audit` is network-free by default even if `ANTHROPIC_API_KEY` exists. With an approved reference set and benchmark snapshot it runs the production extractors, filters, programmatic checks, slicers, and unchanged prompts to create a uniquely identified evaluation run. Planning requires a positive @@ -311,7 +311,7 @@ Keep the printed `.model-evaluation/runs/` path. The summary shows the model, endpoint, reasoning effort, benchmark and reference identities, prompt request set, estimated input usage, retry policy, saved cost guardrail, and destination. A billable run requires `--live`, that run directory, and -`OPENAI_API_KEY`. The command revalidates the frozen evidence, prints the same +`ANTHROPIC_API_KEY`. The command revalidates the frozen evidence, prints the same summary, and asks a terminal operator to type `yes`: ```bash @@ -324,9 +324,17 @@ Deliberate automation may add `--auto-approve`. It skips terminal input but does not skip the summary, API credential, integrity checks, saved cost limit, or overwrite protection. Any rerun requires planning a fresh evaluation run. -The runner uses `gpt-5.6-luna` through Chat Completions with medium reasoning, -omits temperature, keeps summaries disabled, and preserves every attempt and -raw response. It retries only transient timeout, rate-limit, and server +The runner uses pinned `claude-haiku-4-5-20251001` through Anthropic Messages +with manual extended thinking capped at 16,000 tokens and a total output cap +of 24,192 tokens (up to 8,192 for final text after the full thinking budget). +It omits temperature, keeps summaries disabled, and streams internally. Only +final audit text is retained; thinking, signatures, and redacted thinking are +discarded. Aggregate output usage includes billed thinking; a separate reasoning +token count is unavailable. `pricing.v2.json` records the official direct API +rates ($1/M input, $5/M output), source, and verification date. The plan and +approval summary display the configuration, pricing identity, and cost guardrail. +Historical Luna runs remain readable, but cannot be executed under this target. +It preserves every attempt and final response. It retries only transient timeout, rate-limit, and server failures (at most twice). Exhausted failures make a run incomplete and unrankable; successful malformed JSON is retained as a format failure and is not retried. diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index 95a4a8d..e78011b 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from enum import Enum from typing import Any, Literal +from types import SimpleNamespace from .client import is_gemini_model, is_openai_model, supports_temperature @@ -34,6 +35,7 @@ class AuditRequestConfig: reasoning_effort: ReasoningEffort | None = None temperature: float | None = 0.1 max_output_tokens: int = 8192 + thinking_budget_tokens: int | None = None def __post_init__(self) -> None: """Reject configuration values that cannot form an audit request.""" @@ -42,10 +44,28 @@ def __post_init__(self) -> None: if self.max_output_tokens <= 0: raise ValueError("max_output_tokens must be greater than zero") + if self.thinking_budget_tokens is not None: + if is_openai_model(self.model) or is_gemini_model(self.model): + raise ValueError("thinking_budget_tokens requires Anthropic") + if not 1024 <= self.thinking_budget_tokens < self.max_output_tokens: + raise ValueError("thinking budget must be >= 1024 and below output cap") + if self.temperature is not None: + raise ValueError("temperature must be omitted with thinking") + def as_metadata(self) -> dict[str, Any]: """Return the requested configuration as serializable metadata.""" omitted = [] if self.temperature is not None else ["temperature"] return { + **( + { + "thinking": { + "type": "enabled", + "budget_tokens": self.thinking_budget_tokens, + } + } + if self.thinking_budget_tokens is not None + else {} + ), "model": self.model, "reasoning_effort": self.reasoning_effort, "max_output_tokens": self.max_output_tokens, @@ -102,16 +122,14 @@ def _usage_result( total_tokens: int | None = None, cached_input_tokens: int = 0, cache_creation_input_tokens: int = 0, - reasoning_tokens: int = 0, + reasoning_tokens: int | None = 0, ) -> dict[str, Any]: """Return common token categories plus the lossless provider payload.""" return { "input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": ( - total_tokens - if total_tokens is not None - else input_tokens + output_tokens + total_tokens if total_tokens is not None else input_tokens + output_tokens ), "cached_input_tokens": cached_input_tokens, "cache_creation_input_tokens": cache_creation_input_tokens, @@ -221,18 +239,24 @@ def _provider_identity(self) -> dict[str, Any]: def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: """Send one prompt through Anthropic Messages.""" - request_kwargs = { + request_kwargs: dict[str, Any] = { "model": self.model, "max_tokens": self.max_tokens, "messages": [{"role": "user", "content": prompt}], } - if ( - self.request_config.temperature is not None - and supports_temperature(self.model) + if self.request_config.temperature is not None and supports_temperature( + self.model ): request_kwargs["temperature"] = self.request_config.temperature - message = self._client.messages.create(**request_kwargs) + if self.request_config.thinking_budget_tokens is not None: + request_kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": self.request_config.thinking_budget_tokens, + } + message = self._stream_anthropic(request_kwargs) + else: + message = self._client.messages.create(**request_kwargs) response_text = "".join( block.text for block in message.content if block.type == "text" ) @@ -250,11 +274,10 @@ def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: }, "usage": _usage_result( message.usage, + reasoning_tokens=None, input_tokens=message.usage.input_tokens, output_tokens=message.usage.output_tokens, - cached_input_tokens=getattr( - message.usage, "cache_read_input_tokens", 0 - ) + cached_input_tokens=getattr(message.usage, "cache_read_input_tokens", 0) or 0, cache_creation_input_tokens=getattr( message.usage, "cache_creation_input_tokens", 0 @@ -266,9 +289,49 @@ def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: "failure": None, } + def _stream_anthropic(self, request_kwargs: dict[str, Any]) -> Any: + """Consume Messages events without accumulating private thinking blocks.""" + text_parts: list[str] = [] + usage: dict[str, Any] = {} + message = None + stopped = False + with self._client.messages.create(**request_kwargs, stream=True) as stream: + for event in stream: + if event.type == "message_start": + source = event.message + message = SimpleNamespace( + id=source.id, + model=source.model, + type=source.type, + stop_reason=None, + ) + usage.update(_provider_data(source.usage)) + elif event.type == "content_block_start": + if event.content_block.type == "text": + text_parts.append(event.content_block.text) + elif event.type == "content_block_delta": + if event.delta.type == "text_delta": + text_parts.append(event.delta.text) + elif event.type == "message_delta" and message is not None: + message.stop_reason = event.delta.stop_reason + usage.update( + { + key: value + for key, value in _provider_data(event.usage).items() + if value is not None + } + ) + elif event.type == "message_stop": + stopped = True + if message is None or not stopped: + raise ValueError("Anthropic stream ended before message_stop") + message.content = [SimpleNamespace(type="text", text="".join(text_parts))] + message.usage = SimpleNamespace(**usage) + return message + def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: """Send one prompt through OpenAI Chat Completions.""" - request_kwargs = { + request_kwargs: dict[str, Any] = { "model": self.model, "messages": [{"role": "user", "content": prompt}], } @@ -276,17 +339,13 @@ def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: request_kwargs["max_tokens"] = self.max_tokens else: request_kwargs["max_completion_tokens"] = self.max_tokens - request_kwargs["reasoning_effort"] = ( - self.request_config.reasoning_effort - ) + request_kwargs["reasoning_effort"] = self.request_config.reasoning_effort if self.request_config.temperature is not None: request_kwargs["temperature"] = self.request_config.temperature response = self._client.chat.completions.create(**request_kwargs) prompt_details = getattr(response.usage, "prompt_tokens_details", None) - completion_details = getattr( - response.usage, "completion_tokens_details", None - ) + completion_details = getattr(response.usage, "completion_tokens_details", None) return { "success": True, @@ -298,22 +357,15 @@ def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: "response_id": getattr(response, "id", None), "response_model": response.model, "service_tier": getattr(response, "service_tier", None), - "system_fingerprint": getattr( - response, "system_fingerprint", None - ), + "system_fingerprint": getattr(response, "system_fingerprint", None), }, "usage": _usage_result( response.usage, input_tokens=response.usage.prompt_tokens, output_tokens=response.usage.completion_tokens, total_tokens=getattr(response.usage, "total_tokens", None), - cached_input_tokens=getattr( - prompt_details, "cached_tokens", 0 - ) - or 0, - reasoning_tokens=getattr( - completion_details, "reasoning_tokens", 0 - ) + cached_input_tokens=getattr(prompt_details, "cached_tokens", 0) or 0, + reasoning_tokens=getattr(completion_details, "reasoning_tokens", 0) or 0, ), "stop_reason": response.choices[0].finish_reason, @@ -323,7 +375,7 @@ def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: """Send one prompt through Gemini generateContent.""" - generation_config = {"max_output_tokens": self.max_tokens} + generation_config: dict[str, Any] = {"max_output_tokens": self.max_tokens} if self.request_config.temperature is not None: generation_config["temperature"] = self.request_config.temperature @@ -342,22 +394,16 @@ def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: "name": "google", "endpoint": "models.generate_content", "response_id": getattr(response, "response_id", None), - "response_model": getattr( - response, "model_version", self.model - ), + "response_model": getattr(response, "model_version", self.model), }, "usage": _usage_result( usage, input_tokens=usage.prompt_token_count, output_tokens=usage.candidates_token_count, total_tokens=getattr(usage, "total_token_count", None), - cached_input_tokens=getattr( - usage, "cached_content_token_count", 0 - ) + cached_input_tokens=getattr(usage, "cached_content_token_count", 0) or 0, - reasoning_tokens=( - getattr(usage, "thoughts_token_count", 0) or 0 - ), + reasoning_tokens=(getattr(usage, "thoughts_token_count", 0) or 0), ), "stop_reason": ( response.candidates[0].finish_reason.name diff --git a/pyproject.toml b/pyproject.toml index 2ec595d..c3e4ce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.v1.json"] +"vision_aid.evaluation" = ["pricing.v1.json", "pricing.v2.json"] "processing_scripts.llm" = ["*.txt"] diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index 15faaf8..ab486b3 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -5,6 +5,8 @@ from types import SimpleNamespace from unittest import TestCase +import pytest + from entry_points.run_pipeline import ( AuditRequestConfig, PipelineClient, @@ -110,9 +112,7 @@ def test_luna_request_uses_medium_reasoning_without_temperature(self): "model": "gpt-5.6-luna", "max_completion_tokens": 8192, "reasoning_effort": "medium", - "messages": [ - {"role": "user", "content": "audit this payload"} - ], + "messages": [{"role": "user", "content": "audit this payload"}], } ], ) @@ -186,9 +186,7 @@ def test_legacy_provider_request_defaults_are_preserved(self): ) self.assertEqual(anthropic_result["provider"]["name"], "anthropic") self.assertEqual(anthropic_result["usage"]["cached_input_tokens"], 7) - self.assertEqual( - anthropic_result["usage"]["cache_creation_input_tokens"], 3 - ) + self.assertEqual(anthropic_result["usage"]["cache_creation_input_tokens"], 3) self.assertEqual( anthropic_result["request"]["sampling"], { @@ -216,9 +214,7 @@ def test_legacy_provider_request_defaults_are_preserved(self): ) openai_fake = OpenAIFake(openai_response) - PipelineClient("unused", "gpt-4o", provider_client=openai_fake).call( - "audit" - ) + PipelineClient("unused", "gpt-4o", provider_client=openai_fake).call("audit") self.assertEqual( openai_fake.chat.completions.requests, @@ -243,11 +239,7 @@ def test_legacy_provider_request_defaults_are_preserved(self): cached_content_token_count=2, thoughts_token_count=2, ), - candidates=[ - SimpleNamespace( - finish_reason=SimpleNamespace(name="STOP") - ) - ], + candidates=[SimpleNamespace(finish_reason=SimpleNamespace(name="STOP"))], ) gemini_fake = GeminiFake(gemini_response) @@ -275,6 +267,7 @@ def test_legacy_provider_request_defaults_are_preserved(self): def test_failure_exposes_request_and_provider_information(self): """Failures retain the metadata needed for later classification.""" + class ProviderFailure(Exception): status_code = 429 code = "rate_limit_exceeded" @@ -348,3 +341,75 @@ def test_pipeline_persists_explicit_request_configuration(self): self.assertEqual(manifest["request_config"], config.as_metadata()) self.assertEqual(manifest["total_input_tokens"], 0) self.assertEqual(manifest["total_output_tokens"], 0) + + +@pytest.mark.parametrize( + "parts,stop", [(["[", "]"], "end_turn"), ([], "end_turn"), (["["], "max_tokens")] +) +def test_haiku_thinking_stream_exposes_only_final_text(parts, stop): + from contextlib import nullcontext + from processing_scripts.llm_client.audit import AuditRequestClient + + def event(kind, **values): + return SimpleNamespace(type=kind, **values) + + events = [ + event( + "message_start", + message=SimpleNamespace( + id="msg-thinking", + model="claude-haiku-4-5-20251001", + type="message", + usage=SimpleNamespace(input_tokens=100, output_tokens=1), + ), + ), + event( + "content_block_start", content_block=event("thinking", thinking="SECRET") + ), + event("content_block_delta", delta=event("thinking_delta", thinking="SECRET")), + event( + "content_block_delta", delta=event("signature_delta", signature="SECRET") + ), + event( + "content_block_start", + content_block=event("redacted_thinking", data="SECRET"), + ), + *[ + event("content_block_start", content_block=event("text", text=part)) + for part in parts + ], + event("content_block_delta", delta=event("text_delta", text="")), + event( + "message_delta", + delta=SimpleNamespace(stop_reason=stop), + usage=SimpleNamespace(output_tokens=16002), + ), + event("message_stop"), + ] + fake = AnthropicFake(nullcontext(iter(events))) + result = AuditRequestClient( + "unused", + provider_client=fake, + request_config=AuditRequestConfig( + model="claude-haiku-4-5-20251001", + temperature=None, + thinking_budget_tokens=16000, + max_output_tokens=24192, + ), + ).call("audit") + assert fake.messages.requests == [ + { + "model": "claude-haiku-4-5-20251001", + "max_tokens": 24192, + "messages": [{"role": "user", "content": "audit"}], + "thinking": {"type": "enabled", "budget_tokens": 16000}, + "stream": True, + } + ] + assert result["success"] + assert result["response"] == "".join(parts) + assert result["stop_reason"] == stop + assert result["usage"]["output_tokens"] == 16002 + assert result["usage"]["input_tokens"] == 100 + assert result["usage"]["reasoning_tokens"] is None + assert "SECRET" not in str(result) diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py index 3128874..3eb463b 100644 --- a/tests/test_evaluation_audit.py +++ b/tests/test_evaluation_audit.py @@ -51,16 +51,14 @@ def _fake_pipeline( assert api_key is None assert dry_run is True assert include_summaries is False - assert model == "gpt-5.6-luna" - assert request_config.reasoning_effort == "medium" + assert model == "claude-haiku-4-5-20251001" + assert request_config.thinking_budget_tokens == 16000 assert request_config.temperature is None - assert request_config.max_output_tokens == 8192 + assert request_config.max_output_tokens == 24192 prompts = output_dir / "prompts" prompts.mkdir(parents=True) entries = [] - for index, name in enumerate( - ("heading_structure", "link_clarity"), start=1 - ): + for index, name in enumerate(("heading_structure", "link_clarity"), start=1): prompt_text = f"prompt {index}" (prompts / f"{name}.json").write_text( json.dumps( @@ -85,9 +83,7 @@ def _fake_pipeline( "status": "dry_run", } ) - (output_dir / "programmatic_findings.json").write_text( - "[]", encoding="utf-8" - ) + (output_dir / "programmatic_findings.json").write_text("[]", encoding="utf-8") return { "html_file": html_path, "model": model, @@ -105,9 +101,7 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( ) -> None: """A dry plan records the complete frozen benchmark configuration.""" snapshot = tmp_path / "source.html" - snapshot.write_text( - "Synthetic", encoding="utf-8" - ) + snapshot.write_text("Synthetic", encoding="utf-8") snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() run_directory = tmp_path / "run" @@ -125,11 +119,12 @@ def test_dry_plan_freezes_configuration_content_and_code_identities( ) assert plan["configuration"] == { - "endpoint": "chat.completions", + "endpoint": "messages", "include_summaries": False, - "max_output_tokens": 8192, - "model": "gpt-5.6-luna", - "reasoning_effort": "medium", + "max_output_tokens": 24192, + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic", + "thinking": {"type": "enabled", "budget_tokens": 16000}, "temperature": None, "execution": "sequential", } @@ -178,20 +173,18 @@ def call(self, prompt: str) -> dict: return self.outcomes.pop(0) -def _success( - response: str, input_tokens: int = 10, output_tokens: int = 5 -) -> dict: +def _success(response: str, input_tokens: int = 10, output_tokens: int = 5) -> dict: """Return one successful synthetic provider result.""" return { "success": True, "response": response, - "model": "gpt-5.6-luna", - "provider": {"name": "openai", "endpoint": "chat.completions"}, + "model": "claude-haiku-4-5-20251001", + "provider": {"name": "anthropic", "endpoint": "messages"}, "usage": { "input_tokens": input_tokens, "cached_input_tokens": 0, "output_tokens": output_tokens, - "reasoning_tokens": 2, + "reasoning_tokens": None, "cache_creation_input_tokens": 0, "total_tokens": input_tokens + output_tokens, "provider_usage": {}, @@ -207,8 +200,8 @@ def _transient() -> dict: return { "success": False, "response": None, - "model": "gpt-5.6-luna", - "provider": {"name": "openai", "endpoint": "chat.completions"}, + "model": "claude-haiku-4-5-20251001", + "provider": {"name": "anthropic", "endpoint": "messages"}, "usage": { "input_tokens": 2, "cached_input_tokens": 0, @@ -301,7 +294,8 @@ def test_live_gates_retries_and_format_failures( assert len(manifest["requests"][0]["attempts"]) == 2 assert manifest["format_failures"] == ["link_clarity"] assert manifest["usage"]["input_tokens"] == 22 - assert manifest["estimated_audit_cost_usd"] == "0.0000164" + assert manifest["usage"]["reasoning_tokens"] is None + assert manifest["estimated_audit_cost_usd"] == "0.000072" assert manifest["configuration"]["temperature"] is None assert manifest["approval"] == { "mode": "interactive", @@ -584,3 +578,17 @@ def test_versioned_price_schedule_uses_reported_token_categories() -> None: ) == "0.0000298" ) + + +def test_haiku_prices_all_output_including_thinking(): + assert ( + PriceSchedule.default().estimate( + "claude-haiku-4-5-20251001", + { + "input_tokens": 100, + "output_tokens": 16002, + "reasoning_tokens": None, + }, + ) + == "0.08011" + ) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 9fd2747..706a745 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -27,9 +27,7 @@ b'\r\nPristine' b"caf\xc3\xa9\r\n" ) -SNAPSHOT_SHA256 = ( - "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" -) +SNAPSHOT_SHA256 = "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" class SnapshotHTTPServer(ThreadingHTTPServer): @@ -444,9 +442,7 @@ def publish_competing_snapshot() -> None: assert result.returncode == 2 assert "appeared during capture" in result.stderr - assert [path.name for path in snapshot_directory.iterdir()] == [ - "race-marker" - ] + assert [path.name for path in snapshot_directory.iterdir()] == ["race-marker"] def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( @@ -464,15 +460,10 @@ def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( ) assert result.returncode == 2 - assert ( - "DAT Vision Aid fixture is not a Pristine benchmark input" - in result.stderr - ) + assert "DAT Vision Aid fixture is not a Pristine benchmark input" in result.stderr -@pytest.mark.parametrize( - "api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"] -) +@pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( tmp_path: Path, synthetic_workbook: Path, @@ -544,16 +535,11 @@ def call(self, _prompt: str) -> dict[str, object]: "duration_seconds": 0, } - monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") monkeypatch.setattr(sys, "stdin", TerminalInput(" YeS \n")) monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) - assert ( - evaluation_main( - ["audit", "--live", "--run-dir", str(run_directory)] - ) - == 0 - ) + assert evaluation_main(["audit", "--live", "--run-dir", str(run_directory)]) == 0 output = capsys.readouterr().out manifest = json.loads( @@ -590,7 +576,7 @@ class ForbiddenClient: def __init__(self, **_kwargs: object) -> None: raise AssertionError("provider client was constructed") - monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") monkeypatch.setattr( sys, "stdin", @@ -599,9 +585,7 @@ def __init__(self, **_kwargs: object) -> None: monkeypatch.setattr(evaluation_audit, "AuditRequestClient", ForbiddenClient) with pytest.raises(SystemExit) as raised: - evaluation_main( - ["audit", "--live", "--run-dir", str(run_directory)] - ) + evaluation_main(["audit", "--live", "--run-dir", str(run_directory)]) assert raised.value.code == 2 captured = capsys.readouterr() @@ -628,15 +612,13 @@ def test_live_option_combinations_keep_every_spending_gate_mandatory( with pytest.raises(SystemExit): evaluation_main(["audit", "--max-audit-cost-usd", "0"]) assert "greater than zero" in capsys.readouterr().err - assert { - path.name for path in (workspace.root / "runs").iterdir() - } == existing_runs + assert {path.name for path in (workspace.root / "runs").iterdir()} == existing_runs with pytest.raises(SystemExit): evaluation_main(["audit", "--auto-approve"]) assert "only with --live" in capsys.readouterr().err - monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") with pytest.raises(SystemExit): evaluation_main( [ @@ -651,7 +633,8 @@ def test_live_option_combinations_keep_every_spending_gate_mandatory( ) assert "reuses the saved cost guardrail" in capsys.readouterr().err - monkeypatch.delenv("OPENAI_API_KEY") + monkeypatch.delenv("ANTHROPIC_API_KEY") + monkeypatch.setenv("OPENAI_API_KEY", "wrong-provider-key") with pytest.raises(SystemExit): evaluation_main( [ @@ -662,7 +645,7 @@ def test_live_option_combinations_keep_every_spending_gate_mandatory( "--auto-approve", ] ) - assert "requires OPENAI_API_KEY" in capsys.readouterr().err + assert "requires ANTHROPIC_API_KEY" in capsys.readouterr().err assert not (run_directory / "live-manifest.json").exists() @@ -787,7 +770,7 @@ def call(self, _prompt: str) -> dict[str, object]: } monkeypatch.chdir(tmp_path) - monkeypatch.setenv("OPENAI_API_KEY", "synthetic-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) assert ( evaluation_main( @@ -803,9 +786,9 @@ def call(self, _prompt: str) -> dict[str, object]: ) live_output = capsys.readouterr().out for expected in ( - "Model: gpt-5.6-luna", - "Endpoint: chat.completions", - "Reasoning effort: medium", + "Model: claude-haiku-4-5-20251001", + "Endpoint: messages", + "Thinking: enabled; budget tokens: 16000", "Benchmark snapshot SHA-256:", "Reference set:", "Prompts (", @@ -846,9 +829,7 @@ def call(self, _prompt: str) -> dict[str, object]: for name, original_bytes in live_artifacts.items(): assert (run_directory / name).read_bytes() == original_bytes - snapshot_path = ( - workspace / "snapshots" / "pristine-homepage" / "source.html" - ) + snapshot_path = workspace / "snapshots" / "pristine-homepage" / "source.html" snapshot_path.unlink() finding_path = run_directory / "canonical-audit-findings.json" @@ -902,9 +883,7 @@ def call(self, _prompt: str) -> dict[str, object]: programmatic_workbook = review_directory / "programmatic-matches.xlsx" programmatic_review = openpyxl.load_workbook(programmatic_workbook) programmatic_matches = programmatic_review["Matches"] - programmatic_columns = { - cell.value: cell.column for cell in programmatic_matches[1] - } + programmatic_columns = {cell.value: cell.column for cell in programmatic_matches[1]} for row in range(2, programmatic_matches.max_row + 1): for name, value in { "decision": "rejected", @@ -926,18 +905,14 @@ def call(self, _prompt: str) -> dict[str, object]: ) assert programmatic_imported.returncode == 0, programmatic_imported.stderr assert (review_directory / "audit-match-decisions.json").is_file() - assert ( - review_directory / "programmatic-match-decisions.json" - ).is_file() + assert (review_directory / "programmatic-match-decisions.json").is_file() live_manifest["format_failures"] = ["synthetic-format"] (run_directory / "live-manifest.json").write_text( json.dumps(live_manifest), encoding="utf-8", ) - reported = run_cli( - "report", "--run-dir", str(run_directory), cwd=tmp_path - ) + reported = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) assert reported.returncode == 0, reported.stderr json_report = workspace / "reports" / f"{run_manifest['run_id']}.json" @@ -945,14 +920,10 @@ def call(self, _prompt: str) -> dict[str, object]: report_data = json.loads(json_report.read_text(encoding="utf-8")) assert report_data["parse_failures"] == ["synthetic-format"] - programmatic_decisions = ( - review_directory / "programmatic-match-decisions.json" - ) + programmatic_decisions = review_directory / "programmatic-match-decisions.json" decision_bytes = programmatic_decisions.read_bytes() programmatic_decisions.unlink() - unresolved = run_cli( - "report", "--run-dir", str(run_directory), cwd=tmp_path - ) + unresolved = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) assert unresolved.returncode == 2 assert "--kind programmatic" in unresolved.stderr assert "--import-matches" in unresolved.stderr @@ -961,9 +932,7 @@ def call(self, _prompt: str) -> dict[str, object]: findings = json.loads(finding_path.read_text(encoding="utf-8")) findings[0]["run_id"] = "different-run" finding_path.write_text(json.dumps(findings), encoding="utf-8") - rejected_finding = run_cli( - "report", "--run-dir", str(run_directory), cwd=tmp_path - ) + rejected_finding = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) assert rejected_finding.returncode == 2 assert "do not belong to the verified run" in rejected_finding.stderr @@ -976,3 +945,64 @@ def call(self, _prompt: str) -> dict[str, object]: ) assert rejected_eligibility.returncode == 2 assert "reference-set" in rejected_eligibility.stderr + + +def test_haiku_plan_freezes_thinking_and_pricing(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr(evaluation_cli, "_PROJECT_ENV_LOADED", True) + run = create_planned_run(tmp_path, monkeypatch, capsys) + plan = evaluation_audit.load_audit_plan(run) + assert plan["configuration"] == { + "model": "claude-haiku-4-5-20251001", + "provider": "anthropic", + "endpoint": "messages", + "thinking": {"type": "enabled", "budget_tokens": 16000}, + "temperature": None, + "max_output_tokens": 24192, + "include_summaries": False, + "execution": "sequential", + } + summary = evaluation_cli._render_execution_summary(plan, run, "dry-run") + for value in ( + "anthropic", + "claude-haiku-4-5-20251001", + "messages", + "16000", + "24192", + "omitted", + plan["pricing_identity"], + "$0.01", + ): + assert value in summary + assert not (run / "live-manifest.json").exists() + + +def test_historical_luna_plan_is_readable_but_cannot_execute( + tmp_path, monkeypatch, capsys +): + from vision_aid.evaluation.serialization import json_digest + + run = create_planned_run(tmp_path, monkeypatch, capsys) + path = run / "evaluation-manifest.json" + plan = json.loads(path.read_text()) + plan["configuration"] = { + "model": "gpt-5.6-luna", + "reasoning_effort": "medium", + "endpoint": "chat.completions", + "temperature": None, + "max_output_tokens": 8192, + "include_summaries": False, + "execution": "sequential", + } + plan.pop("plan_digest") + plan["plan_digest"] = json_digest(plan) + path.write_text(json.dumps(plan)) + assert ( + evaluation_audit.load_audit_plan(run)["configuration"]["model"] + == "gpt-5.6-luna" + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "unused") + with pytest.raises(SystemExit): + evaluation_main(["audit", "--live", "--auto-approve", "--run-dir", str(run)]) + assert "not the frozen Haiku POC" in capsys.readouterr().err + assert not (run / "live-manifest.json").exists() diff --git a/tests/test_scoring_and_reporting.py b/tests/test_scoring_and_reporting.py index e473912..c68666f 100644 --- a/tests/test_scoring_and_reporting.py +++ b/tests/test_scoring_and_reporting.py @@ -72,9 +72,7 @@ def _finding(identifier: str, source: str = "audit") -> CanonicalFinding: ) -def _match( - reference: str, finding: str, subdefect: str | None = None -) -> MatchDecision: +def _match(reference: str, finding: str, subdefect: str | None = None) -> MatchDecision: """Build one accepted, evidence-compatible match decision.""" return MatchDecision( reference_id=reference, @@ -220,7 +218,10 @@ def test_match_conflicts_and_incomplete_runs_cannot_be_ranked() -> None: rank_scores((score,)) -def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: +@pytest.mark.parametrize("reasoning_tokens", [0, None]) +def test_reports_are_projections_of_one_score_object( + tmp_path: Path, reasoning_tokens +) -> None: """Every format exposes the same totals, latency, and row disposition.""" references = ReferenceSet( version="v1", @@ -236,6 +237,7 @@ def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: complete=True, comparable_identity="benchmark-v1", audit_cost_usd="0.001", + usage=Usage(output_tokens=16002, reasoning_tokens=reasoning_tokens), ) score = score_evaluation( references, @@ -252,6 +254,10 @@ def test_reports_are_projections_of_one_score_object(tmp_path: Path) -> None: with paths.csv.open(newline="", encoding="utf-8") as stream: csv_rows = list(csv.DictReader(stream)) markdown = paths.markdown.read_text(encoding="utf-8") + assert json_report["usage"]["reasoning_tokens"] == reasoning_tokens + assert json_report["usage"]["output_tokens"] == 16002 + if reasoning_tokens is None: + assert "Reasoning tokens: unavailable" in markdown assert json_report["workbook_row_recall"]["caught"] == 1 assert json_report["request_duration_sum_seconds"] == 0 assert csv_rows[0]["model_caught"] == "True" diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index c4ead6d..5aa0afa 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -29,12 +29,12 @@ write_json, ) -MODEL = "gpt-5.6-luna" +MODEL = "claude-haiku-4-5-20251001" REQUEST_CONFIG = AuditRequestConfig( model=MODEL, - reasoning_effort="medium", + thinking_budget_tokens=16000, temperature=None, - max_output_tokens=8192, + max_output_tokens=24192, ) RETRYABLE_FAILURES = ("timeout", "rate_limit", "server_error") BACKOFF_SECONDS = (1, 2) @@ -51,10 +51,11 @@ ) FROZEN_CONFIGURATION = { "model": MODEL, - "reasoning_effort": "medium", - "endpoint": "chat.completions", + "provider": "anthropic", + "thinking": {"type": "enabled", "budget_tokens": 16000}, + "endpoint": "messages", "temperature": None, - "max_output_tokens": 8192, + "max_output_tokens": 24192, "include_summaries": False, "execution": "sequential", } @@ -369,7 +370,7 @@ def load_verified_audit_plan( ) if plan.get("configuration") != FROZEN_CONFIGURATION: raise AuditExecutionError( - "The planned configuration is not the frozen Luna POC." + "The planned configuration is not the frozen Haiku POC." ) price_schedule = schedule or PriceSchedule.default() if plan.get("pricing_identity") != price_schedule.identity: @@ -439,7 +440,7 @@ def _failure_classification(result: dict[str, Any]) -> str: return "permanent" -def _empty_usage() -> dict[str, int]: +def _empty_usage() -> dict[str, int | None]: """Create the token-category accumulator used across attempts.""" return { "input_tokens": 0, @@ -450,11 +451,15 @@ def _empty_usage() -> dict[str, int]: } -def _add_usage(total: dict[str, int], result: dict[str, Any]) -> None: +def _add_usage(total: dict[str, int | None], result: dict[str, Any]) -> None: """Add API-reported categories from one possibly billed attempt.""" usage = result.get("usage") or {} for key in total: - total[key] += int(usage.get(key, 0) or 0) + value = usage.get(key, 0) + previous = total[key] + total[key] = ( + None if value is None or previous is None else previous + int(value) + ) def execute_audit_run( @@ -474,7 +479,7 @@ def execute_audit_run( "Live execution requires the explicit --live flag." ) if not api_key: - raise AuditExecutionError("Live execution requires OPENAI_API_KEY.") + raise AuditExecutionError("Live execution requires ANTHROPIC_API_KEY.") if approval_mode not in {"interactive", "auto"}: raise AuditExecutionError( "Live execution requires interactive or automatic approval." diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 7a2a6df..d2d1c8d 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -245,7 +245,12 @@ def _render_execution_summary( "Evaluation execution summary", f" Model: {configuration['model']}", f" Endpoint: {configuration['endpoint']}", - f" Reasoning effort: {configuration['reasoning_effort']}", + f" Provider: {configuration['provider']}", + f" Thinking: {configuration['thinking']['type']}; budget tokens: " + f"{configuration['thinking']['budget_tokens']}", + f" Total output cap: {configuration['max_output_tokens']}", + " Temperature: omitted", + f" Pricing: {plan['pricing_version']} ({plan['pricing_identity']})", f" Benchmark snapshot SHA-256: {plan['snapshot_sha256']}", f" Reference set: {plan['reference_set_version']}", f" Reference identity: {plan['eligibility_identity']}", @@ -312,9 +317,9 @@ def _audit(arguments: argparse.Namespace) -> int: run_directory = arguments.run_dir.resolve() plan = load_verified_audit_plan(run_directory) ensure_live_destination_available(run_directory) - if not os.environ.get("OPENAI_API_KEY"): + if not os.environ.get("ANTHROPIC_API_KEY"): raise PrivateInputError( - "Live execution requires OPENAI_API_KEY in addition to " + "Live execution requires ANTHROPIC_API_KEY in addition to " "approval." ) approval_mode = "auto" if arguments.auto_approve else "interactive" @@ -330,7 +335,7 @@ def _audit(arguments: argparse.Namespace) -> int: manifest = execute_audit_run( run_directory, live=True, - api_key=os.environ.get("OPENAI_API_KEY"), + api_key=os.environ.get("ANTHROPIC_API_KEY"), approval_mode=approval_mode, approved_at=approved_at, ) diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index 1a1ebe1..42e7531 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any -DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.v1.json") +DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.v2.json") MILLION = Decimal(1_000_000) @@ -42,7 +42,7 @@ def version(self) -> str: """Return the curator-assigned price-schedule version.""" return str(self.data["version"]) - def estimate(self, model: str, usage: Mapping[str, int]) -> str: + def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: """Price reported categories without rounding away tie precision.""" try: rates = self.data["models"][model] @@ -50,12 +50,12 @@ def estimate(self, model: str, usage: Mapping[str, int]) -> str: raise ValueError( f"No versioned pricing exists for model {model!r}." ) from error - input_tokens = Decimal(usage.get("input_tokens", 0)) - cached_tokens = Decimal(usage.get("cached_input_tokens", 0)) + input_tokens = Decimal(usage.get("input_tokens", 0) or 0) + cached_tokens = Decimal(usage.get("cached_input_tokens", 0) or 0) cache_creation_tokens = Decimal( - usage.get("cache_creation_input_tokens", 0) + usage.get("cache_creation_input_tokens", 0) or 0 ) - output_tokens = Decimal(usage.get("output_tokens", 0)) + output_tokens = Decimal(usage.get("output_tokens", 0) or 0) uncached_tokens = max(Decimal(0), input_tokens - cached_tokens) input_multiplier = Decimal(1) output_multiplier = Decimal(1) diff --git a/vision_aid/evaluation/pricing.v2.json b/vision_aid/evaluation/pricing.v2.json new file mode 100644 index 0000000..e15bca5 --- /dev/null +++ b/vision_aid/evaluation/pricing.v2.json @@ -0,0 +1,24 @@ +{ + "models": { + "gpt-5.6-luna": { + "cache_creation_input_per_million_usd": "0.25", + "cached_input_per_million_usd": "0.02", + "input_per_million_usd": "0.20", + "long_context_input_multiplier": "2", + "long_context_output_multiplier": "1.5", + "long_context_threshold_tokens": 272000, + "output_per_million_usd": "1.20", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", + "published_at": "2026-07-09" + }, + "claude-haiku-4-5-20251001": { + "input_per_million_usd": "1", + "output_per_million_usd": "5", + "cached_input_per_million_usd": "0.10", + "cache_creation_input_per_million_usd": "1.25" + } + }, + "published_at": "2026-09-22", + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "version": "anthropic-public-2026-09-22" +} diff --git a/vision_aid/evaluation/reporting.py b/vision_aid/evaluation/reporting.py index caf935a..ce45cd6 100644 --- a/vision_aid/evaluation/reporting.py +++ b/vision_aid/evaluation/reporting.py @@ -133,7 +133,11 @@ def write_reports( f"- Input tokens: {score.usage['input_tokens']}", f"- Cached input tokens: {score.usage['cached_input_tokens']}", f"- Output tokens: {score.usage['output_tokens']}", - f"- Reasoning tokens: {score.usage['reasoning_tokens']}", + "- Reasoning tokens: " + ( + "unavailable (included in aggregate output)" + if score.usage['reasoning_tokens'] is None + else str(score.usage['reasoning_tokens']) + ), ( "- Cache-creation input tokens: " f"{score.usage['cache_creation_input_tokens']}" diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index 85e956e..61cb710 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -103,7 +103,7 @@ class Usage: input_tokens: int = 0 cached_input_tokens: int = 0 output_tokens: int = 0 - reasoning_tokens: int = 0 + reasoning_tokens: int | None = 0 cache_creation_input_tokens: int = 0 diff --git a/vision_aid/evaluation/scoring.py b/vision_aid/evaluation/scoring.py index 79632df..68c9005 100644 --- a/vision_aid/evaluation/scoring.py +++ b/vision_aid/evaluation/scoring.py @@ -64,7 +64,7 @@ class EvaluationScore: audit_cost_usd: str evaluation_cost_usd: str total_experiment_cost_usd: str - usage: dict[str, int] + usage: dict[str, int | None] wall_time_seconds: float request_durations_seconds: tuple[float, ...] request_duration_sum_seconds: float diff --git a/vision_aid/evaluation/serialization.py b/vision_aid/evaluation/serialization.py index aeb1aae..3f14ad6 100644 --- a/vision_aid/evaluation/serialization.py +++ b/vision_aid/evaluation/serialization.py @@ -61,12 +61,8 @@ def load_reference_set(path: Path) -> ReferenceSet: for value in data["references"]: value = dict(value) value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) - value["required_subdefects"] = tuple( - value.get("required_subdefects", ()) - ) - value["eligibility_review"] = _provenance( - value.get("eligibility_review") - ) + value["required_subdefects"] = tuple(value.get("required_subdefects", ())) + value["eligibility_review"] = _provenance(value.get("eligibility_review")) references.append(ReferenceDefect(**value)) return ReferenceSet( version=data["version"], @@ -92,9 +88,7 @@ def load_findings(path: Path) -> tuple[CanonicalFinding, ...]: return tuple(findings) -def save_match_decisions( - path: Path, decisions: tuple[MatchDecision, ...] -) -> None: +def save_match_decisions(path: Path, decisions: tuple[MatchDecision, ...]) -> None: """Persist reviewed match decisions.""" write_json(path, [asdict(item) for item in decisions]) @@ -145,30 +139,27 @@ def load_verified_run_metadata( "Run manifest benchmark identities are inconsistent or changed." ) if manifest["workbook_sha256"] != reference_set.workbook_sha256: - raise ValueError( - "Run manifest workbook checksum does not match references." - ) + raise ValueError("Run manifest workbook checksum does not match references.") if manifest["reference_set_version"] != reference_set.version: raise ValueError( "Run manifest reference-set version does not match references." ) expected_eligibility = reference_set_identity(reference_set) if manifest["eligibility_identity"] != expected_eligibility: - raise ValueError( - "Run manifest eligibility decisions do not match references." - ) + raise ValueError("Run manifest eligibility decisions do not match references.") usage_value = manifest.get("usage", {}) + reasoning_tokens = usage_value.get("reasoning_tokens", 0) usage = Usage( + reasoning_tokens=None if reasoning_tokens is None else int(reasoning_tokens), **{ key: int(usage_value.get(key, 0)) for key in ( "input_tokens", "cached_input_tokens", "output_tokens", - "reasoning_tokens", "cache_creation_input_tokens", ) - } + }, ) return RunMetadata( run_id=str(manifest["run_id"]), @@ -180,8 +171,7 @@ def load_verified_run_metadata( usage=usage, wall_time_seconds=float(manifest.get("wall_time_seconds", 0)), request_durations_seconds=tuple( - float(value) - for value in manifest.get("request_durations_seconds", ()) + float(value) for value in manifest.get("request_durations_seconds", ()) ), workbook_sha256=str(manifest["workbook_sha256"]), snapshot_sha256=str(manifest["snapshot_sha256"]), From ffec10d280ed1205c05b9b039f9f12deb08b685d Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 22 Sep 2026 14:27:03 -0400 Subject: [PATCH 028/126] fix(evaluation): retain reported usage from interrupted streams --- README.md | 2 +- processing_scripts/llm_client/audit.py | 100 ++++++++++++++++--------- tests/test_audit_request.py | 40 ++++++++++ 3 files changed, 107 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index f13c1ea..931e284 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ uv run visionaid-evaluate audit \ ``` Keep the printed `.model-evaluation/runs/` path. The summary shows the -model, endpoint, reasoning effort, benchmark and reference identities, prompt +model, endpoint, thinking budget and total output cap, benchmark and reference identities, prompt request set, estimated input usage, retry policy, saved cost guardrail, and destination. A billable run requires `--live`, that run directory, and `ANTHROPIC_API_KEY`. The command revalidates the frozen evidence, prints the same diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index e78011b..d22cb19 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -138,6 +138,18 @@ def _usage_result( } +class _StreamFailure(Exception): + """Carry observed usage across a transport failure without response content.""" + + def __init__( + self, cause: Exception, usage: dict[str, Any], stop_reason: str | None + ): + super().__init__(str(cause)) + self.cause = cause + self.usage = usage + self.stop_reason = stop_reason + + class AuditRequestClient: """Send an audit prompt through Anthropic, OpenAI, or Gemini. @@ -206,6 +218,12 @@ def call(self, prompt: str) -> dict[str, Any]: result["request"] = resolved_request_metadata(self.request_config) return result except Exception as exc: + partial_usage = {} + stop_reason = None + if isinstance(exc, _StreamFailure): + partial_usage = exc.usage + stop_reason = exc.stop_reason + exc = exc.cause return { "success": False, "response": None, @@ -213,11 +231,20 @@ def call(self, prompt: str) -> dict[str, Any]: "provider": self._provider_identity(), "request": resolved_request_metadata(self.request_config), "usage": _usage_result( - None, - input_tokens=0, - output_tokens=0, + partial_usage or None, + input_tokens=partial_usage.get("input_tokens", 0), + output_tokens=partial_usage.get("output_tokens", 0), + reasoning_tokens=( + 0 if self._is_openai or self._is_gemini else None + ), + cached_input_tokens=partial_usage.get("cache_read_input_tokens", 0) + or 0, + cache_creation_input_tokens=partial_usage.get( + "cache_creation_input_tokens", 0 + ) + or 0, ), - "stop_reason": None, + "stop_reason": stop_reason, "error": str(exc), "duration_seconds": round(time.monotonic() - start, 2), "failure": { @@ -295,36 +322,41 @@ def _stream_anthropic(self, request_kwargs: dict[str, Any]) -> Any: usage: dict[str, Any] = {} message = None stopped = False - with self._client.messages.create(**request_kwargs, stream=True) as stream: - for event in stream: - if event.type == "message_start": - source = event.message - message = SimpleNamespace( - id=source.id, - model=source.model, - type=source.type, - stop_reason=None, - ) - usage.update(_provider_data(source.usage)) - elif event.type == "content_block_start": - if event.content_block.type == "text": - text_parts.append(event.content_block.text) - elif event.type == "content_block_delta": - if event.delta.type == "text_delta": - text_parts.append(event.delta.text) - elif event.type == "message_delta" and message is not None: - message.stop_reason = event.delta.stop_reason - usage.update( - { - key: value - for key, value in _provider_data(event.usage).items() - if value is not None - } - ) - elif event.type == "message_stop": - stopped = True - if message is None or not stopped: - raise ValueError("Anthropic stream ended before message_stop") + try: + with self._client.messages.create(**request_kwargs, stream=True) as stream: + for event in stream: + if event.type == "message_start": + source = event.message + message = SimpleNamespace( + id=source.id, + model=source.model, + type=source.type, + stop_reason=None, + ) + usage.update(_provider_data(source.usage)) + elif event.type == "content_block_start": + if event.content_block.type == "text": + text_parts.append(event.content_block.text) + elif event.type == "content_block_delta": + if event.delta.type == "text_delta": + text_parts.append(event.delta.text) + elif event.type == "message_delta" and message is not None: + message.stop_reason = event.delta.stop_reason + usage.update( + { + key: value + for key, value in _provider_data(event.usage).items() + if value is not None + } + ) + elif event.type == "message_stop": + stopped = True + if message is None or not stopped: + raise ValueError("Anthropic stream ended before message_stop") + except Exception as exc: + raise _StreamFailure( + exc, usage, message.stop_reason if message is not None else None + ) from exc message.content = [SimpleNamespace(type="text", text="".join(text_parts))] message.usage = SimpleNamespace(**usage) return message diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index ab486b3..ef80573 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -413,3 +413,43 @@ def event(kind, **values): assert result["usage"]["input_tokens"] == 100 assert result["usage"]["reasoning_tokens"] is None assert "SECRET" not in str(result) + + +def test_interrupted_thinking_stream_keeps_reported_billed_usage(): + from contextlib import nullcontext + from processing_scripts.llm_client.audit import AuditRequestClient + + def events(): + yield SimpleNamespace( + type="message_start", + message=SimpleNamespace( + id="partial", + model="claude-haiku-4-5-20251001", + type="message", + usage=SimpleNamespace(input_tokens=100, output_tokens=1), + ), + ) + yield SimpleNamespace( + type="message_delta", + delta=SimpleNamespace(stop_reason="max_tokens"), + usage=SimpleNamespace(output_tokens=16002), + ) + raise TimeoutError("stream interrupted") + + result = AuditRequestClient( + "unused", + provider_client=AnthropicFake(nullcontext(events())), + request_config=AuditRequestConfig( + model="claude-haiku-4-5-20251001", + temperature=None, + thinking_budget_tokens=16000, + max_output_tokens=24192, + ), + ).call("audit") + assert not result["success"] + assert result["response"] is None + assert result["usage"]["input_tokens"] == 100 + assert result["usage"]["output_tokens"] == 16002 + assert result["usage"]["reasoning_tokens"] is None + assert result["stop_reason"] == "max_tokens" + assert result["failure"]["type"] == "TimeoutError" From 3e6e9d2e916a62d8abd064ea08ae7c38551c1f82 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 22 Sep 2026 14:27:48 -0400 Subject: [PATCH 029/126] docs(evaluation): resolve Anthropic retarget spec after verification --- .../anthropic-evaluation-retarget/spec.md | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.scratch/anthropic-evaluation-retarget/spec.md b/.scratch/anthropic-evaluation-retarget/spec.md index 1032d56..bca44d1 100644 --- a/.scratch/anthropic-evaluation-retarget/spec.md +++ b/.scratch/anthropic-evaluation-retarget/spec.md @@ -1,4 +1,4 @@ -Status: ready-for-agent +Status: resolved ## Problem Statement @@ -127,3 +127,26 @@ The evaluation CLI will resolve `ANTHROPIC_API_KEY`, display the exact pinned mo - The existing prepared Luna replacement plan cannot be converted in place because model, provider, endpoint, reasoning configuration, and pricing participate in immutable run identity and point-of-use verification. - Model lifecycle should be checked again before a future paid run. The pinned identifier is active at specification time, but a dated model can eventually be deprecated or retired. - This spec supersedes only the Luna-specific target and execution details of the broader model-evaluation proof of concept. Its domain model, benchmark methodology, review separation, scoring rules, and privacy boundaries remain authoritative. + + +## Comments + +Implemented in `96c5a4fe` and review fix `ffec10d2`. + +- [x] Pinned Haiku Messages configuration, thinking stream, final-text-only + evidence, aggregate usage/cost, and unavailable reasoning detail verified. +- [x] Credential, dry-default, approval, guardrail, historical loading and + stale-plan execution rejection covered by offline tests. +- [x] Public provider regression tests, normalization, deterministic scoring, + and reporting pass; 84 tests total. Client/evaluation mypy passes. +- [x] Standards and Spec reviews completed; all findings resolved. +- [x] Fresh private plan `run-20260922T182506-e58810f9df6b` verified unexecuted + with $1 guardrail. Existing benchmark, prompt hashes/order, and eligibility + preserved. Prior run files verified byte-for-byte unchanged. +- [x] All 447 programmatic candidate rows semantically verified before decision + carry-forward; import validated. Private operator log appended with rationale, + configuration, pricing source, superseded Luna plan, and fresh run identity. + +No paid request or live approval occurred. Operational ticket 12 remains open +for the subsequent paid run and review; historical Luna records retain their +original content with supersession notes. From 7b99d1659e29e9b0b50b6b2abff459625bd25781 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Wed, 23 Sep 2026 18:37:24 -0400 Subject: [PATCH 030/126] refactor(evaluation): reduce homepage POC to run and report --- .../anthropic-evaluation-retarget/spec.md | 152 --- .scratch/evaluation-cli-workflow/spec.md | 133 -- ...01-explicit-audit-request-configuration.md | 25 - .../issues/02-private-evaluation-workspace.md | 19 - .../03-freeze-pristine-benchmark-snapshot.md | 20 - .../issues/04-homepage-reference-set.md | 22 - ...5-review-and-score-synthetic-comparison.md | 22 - .../06-programmatic-workbook-coverage.md | 21 - .../issues/07-safe-luna-dry-run.md | 30 - .../issues/08-authorized-live-audit.md | 33 - .../issues/09-normalize-raw-audit-findings.md | 20 - .../issues/10-review-finding-matches.md | 25 - .../11-verify-complete-evaluation-workflow.md | 24 - .../issues/12-run-luna-homepage-poc.md | 29 - .scratch/model-evaluation/progress-report.md | 450 ------- .scratch/model-evaluation/spec.md | 184 --- .scratch/poc-reduction/spec.md | 118 ++ CONTEXT.md | 88 +- README.md | 181 +-- docs/adr/0001-separate-review-from-scoring.md | 2 +- pyproject.toml | 2 +- tests/test_evaluation_audit.py | 594 --------- tests/test_evaluation_cli.py | 1187 +++-------------- tests/test_match_review.py | 100 -- tests/test_programmatic_evaluation.py | 30 - tests/test_reference_set.py | 187 --- tests/test_scoring_and_reporting.py | 374 ------ vision_aid/evaluation/audit.py | 650 +-------- vision_aid/evaluation/cli.py | 672 ++-------- vision_aid/evaluation/matching.py | 404 ------ vision_aid/evaluation/pricing.py | 15 +- vision_aid/evaluation/pricing.v1.json | 16 - vision_aid/evaluation/pricing.v2.json | 11 - vision_aid/evaluation/programmatic.py | 44 - vision_aid/evaluation/references.py | 434 +----- vision_aid/evaluation/reporting.py | 174 --- vision_aid/evaluation/review.py | 156 +++ vision_aid/evaluation/schemas.py | 92 +- vision_aid/evaluation/scoring.py | 327 ----- vision_aid/evaluation/serialization.py | 190 --- vision_aid/evaluation/snapshot.py | 307 ----- vision_aid/evaluation/workspace.py | 116 -- 42 files changed, 798 insertions(+), 6882 deletions(-) delete mode 100644 .scratch/anthropic-evaluation-retarget/spec.md delete mode 100644 .scratch/evaluation-cli-workflow/spec.md delete mode 100644 .scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md delete mode 100644 .scratch/model-evaluation/issues/02-private-evaluation-workspace.md delete mode 100644 .scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md delete mode 100644 .scratch/model-evaluation/issues/04-homepage-reference-set.md delete mode 100644 .scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md delete mode 100644 .scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md delete mode 100644 .scratch/model-evaluation/issues/07-safe-luna-dry-run.md delete mode 100644 .scratch/model-evaluation/issues/08-authorized-live-audit.md delete mode 100644 .scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md delete mode 100644 .scratch/model-evaluation/issues/10-review-finding-matches.md delete mode 100644 .scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md delete mode 100644 .scratch/model-evaluation/issues/12-run-luna-homepage-poc.md delete mode 100644 .scratch/model-evaluation/progress-report.md delete mode 100644 .scratch/model-evaluation/spec.md create mode 100644 .scratch/poc-reduction/spec.md delete mode 100644 tests/test_evaluation_audit.py delete mode 100644 tests/test_match_review.py delete mode 100644 tests/test_programmatic_evaluation.py delete mode 100644 tests/test_reference_set.py delete mode 100644 tests/test_scoring_and_reporting.py delete mode 100644 vision_aid/evaluation/matching.py delete mode 100644 vision_aid/evaluation/pricing.v1.json delete mode 100644 vision_aid/evaluation/programmatic.py delete mode 100644 vision_aid/evaluation/reporting.py create mode 100644 vision_aid/evaluation/review.py delete mode 100644 vision_aid/evaluation/scoring.py delete mode 100644 vision_aid/evaluation/serialization.py delete mode 100644 vision_aid/evaluation/snapshot.py delete mode 100644 vision_aid/evaluation/workspace.py diff --git a/.scratch/anthropic-evaluation-retarget/spec.md b/.scratch/anthropic-evaluation-retarget/spec.md deleted file mode 100644 index bca44d1..0000000 --- a/.scratch/anthropic-evaluation-retarget/spec.md +++ /dev/null @@ -1,152 +0,0 @@ -Status: resolved - -## Problem Statement - -The private accessibility-audit evaluation workflow assumes that its API credential is an OpenAI key and freezes GPT-5.6 Luna at medium reasoning into every planned evaluation run. The available credential is instead an Anthropic key. As a result, the evaluator cannot execute the intended proof of concept even though the repository's shared audit path already supports Anthropic and the public application already defaults to Claude Haiku 4.5. - -The evaluator must be retargeted without turning this focused proof of concept into a general provider-selection system. The replacement must preserve the existing benchmark snapshot, audit prompts, review workflow, deterministic scoring, workbook-row recall ranking, cost reporting, write-once evaluation artifacts, and live-run approval protections. Historical Luna evaluation runs must remain immutable and readable. - -## Solution - -Retarget the private evaluation workflow from GPT-5.6 Luna to the pinned Anthropic model `claude-haiku-4-5-20251001`. Configure every audit request with manual extended thinking capped at 16,000 tokens and a total output cap of 24,192 tokens, leaving up to the existing 8,192-token allowance for the final audit JSON when the full thinking budget is used. Omit temperature and use Anthropic's Messages API. - -Extend the existing shared audit-client boundary only as much as required to send this fixed thinking configuration and consume the streaming response required by the larger output cap. Return the same normalized final-text and usage contract that the evaluator already consumes. Thinking and redacted-thinking blocks will not become evaluation evidence or be persisted; aggregate provider usage will still be retained and priced. - -The evaluation CLI will resolve `ANTHROPIC_API_KEY`, display the exact pinned model, thinking budget, total output cap, endpoint, pricing identity, and cost guardrail in its plan and approval summaries, and continue to be dry by default. The implementation will generate a fresh dry-run plan under a new evaluation run identity. It will not execute a paid live audit. - -## User Stories - -1. As an evaluation operator, I want the private evaluator to use my Anthropic credential, so that I can continue the proof of concept with the credential I was actually issued. -2. As an evaluation operator, I want a missing credential error to name `ANTHROPIC_API_KEY`, so that setup instructions do not send me toward the wrong provider. -3. As an evaluator, I want the audit model pinned to `claude-haiku-4-5-20251001`, so that a moving provider alias cannot silently change an evaluation run. -4. As an evaluator, I want the provider recorded as Anthropic, so that the run manifest accurately identifies the service that generated each audit finding. -5. As an evaluator, I want the endpoint recorded as Anthropic Messages, so that the request protocol is reproducible. -6. As an evaluator, I want Haiku extended thinking enabled, so that the low-cost model has additional reasoning capacity for accessibility judgments. -7. As an evaluator, I want the thinking budget fixed at 16,000 tokens, so that every request in the evaluation run has the same reproducible ceiling. -8. As an evaluator, I want the total output cap fixed at 24,192 tokens, so that a request can use its thinking budget without reducing the prior 8,192-token final-response allowance. -9. As an evaluator, I want temperature omitted while thinking is enabled, so that the request satisfies Anthropic's supported contract. -10. As an evaluation operator, I want the exact thinking and output ceilings shown before execution, so that the potential billed usage is visible. -11. As a budget owner, I want Haiku input and output usage priced from a versioned public price schedule, so that estimated run cost remains reproducible. -12. As a budget owner, I want all billed output tokens included in estimated run cost, including tokens used for thinking, so that cost is not understated. -13. As a budget owner, I want aggregate Anthropic usage preserved without inventing a reasoning-token breakdown that the response does not supply, so that reports distinguish measured values from unavailable detail. -14. As an evaluation operator, I want streaming handled internally by the audit client, so that I receive the same final audit result contract as before. -15. As an evaluator, I want streamed text blocks assembled in provider order, so that the final audit JSON can be normalized correctly. -16. As an evaluator, I want provider stop reasons and aggregate usage retained after streaming, so that format failures, truncation, and cost remain reviewable. -17. As a data owner, I want thinking and redacted-thinking blocks discarded rather than stored, so that internal reasoning does not become benchmark evidence or a new private artifact class. -18. As a comparison reviewer, I want only the model's final audit text normalized into canonical audit findings, so that the review contract remains unchanged. -19. As a comparison reviewer, I want malformed or missing final text handled by the existing normalization policy, so that thinking support does not conceal structured-output failure. -20. As an evaluator, I want the benchmark snapshot, prompts, filters, prompt order, and disabled summaries preserved, so that the provider retarget is the material experimental change. -21. As an evaluator, I want workbook-row recall to remain the primary ranking metric, so that changing providers does not change the definition of model performance. -22. As an evaluator, I want estimated run cost to remain only the tie-breaker, so that the higher per-token price of Haiku does not silently redefine model quality. -23. As an operator, I want the evaluation to remain dry by default even when `ANTHROPIC_API_KEY` exists, so that environment configuration alone cannot spend money. -24. As an operator, I want live execution to retain its explicit flag, positive cost guardrail, and point-of-use live-run approval, so that the provider change cannot bypass spending controls. -25. As an operator, I want this change to create a fresh evaluation run plan, so that its provider, model, thinking, output, and pricing identities are internally consistent. -26. As an operator, I want the fresh plan to remain unexecuted, so that I can review it before authorizing a paid run. -27. As an evaluator, I want the earlier failed Luna run preserved, so that the historical authentication failure remains auditable. -28. As an evaluator, I want the prepared but unexecuted Luna replacement plan preserved and marked as superseded or abandoned, so that immutable run history is not rewritten. -29. As an evaluator, I want old Luna artifacts to remain loadable for reporting and review where supported, so that current configuration changes do not corrupt historical evidence. -30. As an evaluator, I want an old Luna plan rejected for new live execution after the retarget, so that a stale plan cannot run under Haiku credentials or pricing. -31. As a maintainer, I want the public web application and HTTP API left unchanged, so that an evaluator-only retarget does not disturb existing multi-provider behavior. -32. As a maintainer, I want the existing audit-client interface extended rather than duplicated, so that Anthropic request and usage handling remain centralized. -33. As a maintainer, I want thinking support represented explicitly rather than disguised as OpenAI `reasoning_effort`, so that provider-specific semantics remain truthful. -34. As a maintainer, I want streaming enabled only where the configured Anthropic request requires it, so that unrelated audit calls retain their current behavior. -35. As a maintainer, I want existing non-thinking Anthropic calls to keep working, so that the focused evaluator change does not regress production audits. -36. As a maintainer, I want existing OpenAI and Gemini client behavior to remain unchanged, so that this retarget has a narrow regression surface. -37. As a maintainer, I want tests to use fake provider responses, so that verification never requires a credential, network access, or paid tokens. -38. As a maintainer, I want documentation to identify Haiku as the current proof-of-concept target, so that operators do not follow obsolete Luna instructions. -39. As a maintainer, I want historical Luna implementation records retained with supersession notes rather than rewritten, so that the issue tracker remains truthful. -40. As a future evaluator, I want the model identifier and pricing isolated in the existing frozen configuration and price schedule, so that a later retirement can be handled without redesigning the framework. - -## Implementation Decisions - -- This is a fixed-target retarget of the private evaluation workflow, not a general provider-selection feature. -- The pinned model is `claude-haiku-4-5-20251001`; the moving `claude-haiku-4-5` alias will not be used for an evaluation run. -- The request protocol is Anthropic Messages and the credential environment variable is `ANTHROPIC_API_KEY`. -- Manual extended thinking is enabled with `budget_tokens` set to 16,000. Haiku does not support adaptive thinking or an OpenAI-style `medium` reasoning setting. -- The request's `max_tokens` value is 24,192. Thinking and final text share this provider limit; the selected value preserves up to 8,192 tokens for final text when the full thinking budget is consumed. -- Temperature, `top_k`, and `top_p` are omitted from thinking-enabled requests. -- Thinking configuration is modeled explicitly in the shared audit request configuration. It will not overload the OpenAI-specific `reasoning_effort` field. -- The shared audit client gains the minimum streaming behavior required for this Anthropic configuration. It assembles final text, captures the terminal stop reason and usage, and returns the existing provider-neutral audit result. -- Streaming is not introduced as a new public evaluation abstraction. It remains an internal transport detail selected by the Anthropic request requirements. -- Only final text blocks enter normalization. Thinking, signatures, and redacted-thinking blocks are neither returned as audit text nor persisted as raw evaluation evidence. -- Aggregate provider output usage is authoritative for billing. The implementation will not estimate or fabricate a separate Anthropic reasoning-token count. -- Existing usage and reporting schemas will be changed only where needed to avoid presenting unavailable Anthropic reasoning detail as a measured zero. -- Haiku pricing is added to the versioned price schedule using the official direct-API input and output rates current when the schedule is updated. Pricing source and effective date remain recorded. -- Prompt caching and Batch API discounts are not enabled. Their prices may be represented only if the existing schedule requires complete model metadata. -- The frozen evaluation configuration, pricing identity, and resulting benchmark identity change. A new evaluation run is therefore mandatory. -- Previous run directories are immutable. The failed Luna run and the unexecuted Luna replacement plan will not be edited, deleted, or repurposed. -- The approved reference set and benchmark snapshot may be reused because the provider retarget does not change their content. -- Programmatic match decisions may be carried into the fresh run only through the existing row-for-row semantic verification process because run-scoped identifiers change. -- The operator log will append the model-selection rationale, final request configuration, superseded Luna plan status, and fresh dry-run identifier. Historical entries will remain intact. -- Normative evaluation documentation and the currently open operational ticket will be updated for Haiku. Resolved Luna tickets and superseded specifications will receive historical clarification rather than retroactive rewriting. -- Public application behavior is unchanged because it already supports Anthropic and already defaults to the pinned Haiku model. -- The implementation ends after generating and verifying a fresh dry-run plan. It must not make a live Anthropic request. -- No new domain glossary term or ADR is required. The existing concepts of evaluation run, estimated run cost, live-run approval, audit finding, and benchmark snapshot cover the change, while the fixed model target remains a reversible specification decision. - -## Testing Decisions - -- Tests will assert observable request, result, plan, and safety behavior rather than private helper structure. -- Two existing high-level seams are sufficient: the shared audit client's public request/result boundary and the evaluation CLI's dry-run plan boundary. No new test-only seam will be introduced. -- At the audit-client seam, a fake Anthropic stream will verify the pinned model, Messages request shape, 16,000-token thinking budget, 24,192-token total cap, omission of sampling parameters, ordered final-text assembly, terminal stop reason, and aggregate usage. -- At the same seam, mixed thinking and text blocks will verify that only final text is exposed as audit output, while thinking-only or truncated responses produce the existing empty or malformed final-output behavior. -- Existing plain Anthropic tests will remain and prove that non-thinking requests do not accidentally acquire thinking or streaming behavior. -- Existing OpenAI and Gemini request-shape tests will remain unchanged and serve as regression coverage for the provider-specific branch. -- At the evaluation CLI seam, a dry-run test will verify that the manifest and human-readable summary contain the Anthropic provider, pinned model, Messages endpoint, enabled thinking budget, total output cap, omitted temperature, and new pricing identity. -- CLI tests will verify that `ANTHROPIC_API_KEY` is required only for live execution, that an OpenAI-named key is not accepted as the required Anthropic credential, and that dry-run planning requires no credential. -- Safety tests will verify that the explicit live flag, positive cost guardrail, and live-run approval remain mandatory and that no fake or real provider call occurs during the requested dry run. -- Historical-plan tests will verify that an old Luna plan remains readable where historical loading permits but is rejected for execution under the new frozen configuration. -- Pricing tests will calculate Haiku input and aggregate output costs from fixed synthetic usage, including output usage attributable to thinking, without depending on live provider pricing. -- Reporting tests will verify that unavailable provider-specific reasoning detail is not mislabeled as an API-reported zero while total billed output remains correct. -- Normalization and scoring tests should remain provider-neutral. Existing tests for canonical audit findings, malformed responses, deterministic workbook-row recall, and cost-only tie-breaking are prior art and should pass without behavioral changes. -- The complete focused regression suite will cover audit requests, evaluation audit planning/execution policy, evaluation CLI behavior, pricing, reporting, and normalization. No test may access the internet or spend money. -- A good acceptance test demonstrates the complete dry-run behavior through the CLI with fake provider boundaries: it produces a fresh Haiku plan with the exact frozen configuration and no network call. - -## Out of Scope - -- Supporting runtime selection between OpenAI, Anthropic, Gemini, or arbitrary providers in the private evaluator. -- Changing the public website, public HTTP API, or their existing Haiku default. -- Claiming that Claude Haiku 4.5 is officially equivalent to GPT-5.6 Luna at medium reasoning. -- Mapping Anthropic thinking budgets to OpenAI reasoning-effort labels. -- Adaptive thinking, dynamic thinking budgets, per-prompt budgets, retries at a stronger model, or automatic provider fallback. -- Generalizing streaming as a user-selectable mode or changing unrelated non-streaming calls. -- Persisting thinking summaries, encrypted thinking signatures, redacted-thinking blocks, or raw chain-of-thought. -- Changing audit prompts, filters, prompt order, summary behavior, canonical audit-finding normalization, matching rules, or deterministic scoring. -- Enabling prompt caching, Batch API execution, tool use, browser tools, code execution, or computer use. -- Optimizing estimated run cost beyond accurately pricing reported usage. -- Deleting, mutating, or reusing prior Luna evaluation runs. -- Reclassifying reference defects or changing approved match decisions except for the established run-ID carry-forward procedure. -- Executing the paid Haiku audit or granting live-run approval. -- Adding a new domain glossary term or architectural decision record for the provider retarget. - -## Further Notes - -- Claude Haiku 4.5 is the closest current Anthropic cost-first analogue to GPT-5.6 Luna, but this is a product-positioning inference rather than an official cross-provider equivalence. -- At the time of specification, direct Claude API pricing is $1 per million input tokens and $5 per million output tokens. The implementation must verify and record the official price source when updating the versioned schedule. -- Haiku's thinking budget is a ceiling and the model may stop thinking earlier. The provider bills thinking within aggregate output usage. -- The larger total output cap requires streaming under the current Anthropic SDK guidance. This is why a small shared-client change is necessary even though ordinary Anthropic Messages requests are already supported. -- The existing prepared Luna replacement plan cannot be converted in place because model, provider, endpoint, reasoning configuration, and pricing participate in immutable run identity and point-of-use verification. -- Model lifecycle should be checked again before a future paid run. The pinned identifier is active at specification time, but a dated model can eventually be deprecated or retired. -- This spec supersedes only the Luna-specific target and execution details of the broader model-evaluation proof of concept. Its domain model, benchmark methodology, review separation, scoring rules, and privacy boundaries remain authoritative. - - -## Comments - -Implemented in `96c5a4fe` and review fix `ffec10d2`. - -- [x] Pinned Haiku Messages configuration, thinking stream, final-text-only - evidence, aggregate usage/cost, and unavailable reasoning detail verified. -- [x] Credential, dry-default, approval, guardrail, historical loading and - stale-plan execution rejection covered by offline tests. -- [x] Public provider regression tests, normalization, deterministic scoring, - and reporting pass; 84 tests total. Client/evaluation mypy passes. -- [x] Standards and Spec reviews completed; all findings resolved. -- [x] Fresh private plan `run-20260922T182506-e58810f9df6b` verified unexecuted - with $1 guardrail. Existing benchmark, prompt hashes/order, and eligibility - preserved. Prior run files verified byte-for-byte unchanged. -- [x] All 447 programmatic candidate rows semantically verified before decision - carry-forward; import validated. Private operator log appended with rationale, - configuration, pricing source, superseded Luna plan, and fresh run identity. - -No paid request or live approval occurred. Operational ticket 12 remains open -for the subsequent paid run and review; historical Luna records retain their -original content with supersession notes. diff --git a/.scratch/evaluation-cli-workflow/spec.md b/.scratch/evaluation-cli-workflow/spec.md deleted file mode 100644 index 3cd2f34..0000000 --- a/.scratch/evaluation-cli-workflow/spec.md +++ /dev/null @@ -1,133 +0,0 @@ -# Streamline the Private Evaluation CLI Workflow - -Status: resolved - -## Problem Statement - -The private model-evaluation framework has the required safety and reproducibility controls, but its operator workflow is harder than necessary. A live run currently requires copying an authorization digest from a dry run into a later command even though the digest does not establish identity or provide meaningful consent. Match-review imports for audit findings and programmatic findings overwrite the same decision file, and final reporting requires manually constructing a report-bundle JSON document. The resulting walkthrough includes file renames and raw Python that should not be part of an ordinary evaluation run. - -The framework must remain dry by default, preserve its cost guardrail, verify that planned evidence has not drifted, and obtain explicit live-run approval. Those protections should be expressed through a clear point-of-use confirmation and one run-oriented CLI workflow rather than opaque copy-and-paste tokens. - -## Solution - -Make the evaluation run the operator-facing unit of work. Planning creates a unique run directory, records the maximum audit cost, freezes the approved reference set, and prints the billable intent. Live execution receives that run directory, revalidates its internal identities, prints the complete execution summary, and asks an interactive operator to type `yes`. Automation may deliberately replace the prompt with `--auto-approve`, but it receives the same summary and all other spending gates remain mandatory. - -Use the run identifier to organize separate audit and programmatic match-review artifacts. Let reporting discover its verified inputs from the run directory and associated run-scoped reviews. Remove the authorization-digest, plan-path, and report-bundle interfaces so the normal workflow requires neither manual renaming nor custom Python. - -## User Stories - -1. As an evaluation operator, I want a dry run to remain the default, so that an available API key cannot spend money by itself. -2. As an evaluation operator, I want live execution to retain an explicit `--live` flag, so that billable intent is visible in the command. -3. As a budget owner, I want every evaluation run planned with a positive maximum audit cost, so that spending has a defined upper guardrail. -4. As a budget owner, I want the planned cost guardrail frozen with the evaluation run, so that it cannot drift at execution time. -5. As an evaluation operator, I want the live command to reuse the saved cost guardrail, so that I do not need to repeat matching values across commands. -6. As an evaluation operator, I want the complete live execution summary shown immediately before approval, so that I understand what will run and what it may cost. -7. As an evaluation operator, I want the summary to identify the model, endpoint, and reasoning effort, so that model configuration is visible before spending. -8. As an evaluation operator, I want the summary to identify the benchmark snapshot and approved reference set, so that I know which evidence will be evaluated. -9. As an evaluation operator, I want the summary to list the prompt request set and request count, so that approval covers known work. -10. As an evaluation operator, I want the summary to show estimated input usage, retry policy, maximum audit cost, and destination, so that operational consequences are explicit. -11. As an evaluation operator, I want to approve a live run by typing `yes`, so that approval is understandable without interpreting a digest. -12. As an evaluation operator, I want any response other than an exact case-insensitive `yes` to abort, so that ambiguous input cannot start a provider request. -13. As an evaluation operator, I want end-of-input to abort before a provider client is created, so that a missing response cannot be treated as consent. -14. As an evaluation operator, I want interactive approval to require a terminal, so that scripts cannot silently pipe confirmation into the prompt. -15. As an automation author, I want an explicit `--auto-approve` option, so that deliberate non-interactive execution is supported. -16. As an automation author, I want `--auto-approve` accepted only for live execution, so that the option has one clear meaning. -17. As an automation author, I want the full execution summary printed even when approval is automatic, so that logs retain the approved intent. -18. As an auditor, I want the live manifest to record whether approval was interactive or automatic and when it occurred, so that approval provenance is visible without claiming a person's identity. -19. As an evaluation operator, I want live execution to require an API key in addition to approval, so that confirmation alone cannot create a request. -20. As a maintainer, I want internal plan and evidence identities retained, so that modified snapshots, prompts, pricing, eligibility decisions, or configurations still fail closed. -21. As an evaluation operator, I do not want to copy an authorization digest between commands, so that the workflow reflects meaningful consent rather than token matching. -22. As an evaluation operator, I want every planned evaluation run to receive a unique run directory, so that repeated experiments do not collide. -23. As an auditor, I want each evaluation run to freeze the approved reference set it used, so that later curation cannot make old evidence unreportable. -24. As an auditor, I want an evaluation run to retain its immutable model, benchmark, prompt, parser, pricing, and reference identities, so that reports remain reproducible. -25. As an evaluation operator, I want execution to refuse an occupied live-output destination, so that earlier evidence is never overwritten. -26. As an evaluation operator, I want a rerun to require a fresh evaluation run, so that each approval and result remain independently traceable. -27. As a comparison reviewer, I want audit findings and programmatic findings reviewed as distinct kinds, so that their match decisions cannot overwrite one another. -28. As a comparison reviewer, I want match workbooks and validated match decisions scoped by run identifier, so that reviews from different evaluations cannot collide. -29. As a comparison reviewer, I want the review command to infer conventional inputs and outputs from a run directory and review kind, so that ordinary review does not require repeated paths. -30. As a comparison reviewer, I want explicit input and output overrides retained for unusual review cases, so that the simplified workflow does not prevent expert recovery or investigation. -31. As an evaluation operator, I want the report command to accept one run directory, so that I do not manually enumerate scoring inputs. -32. As an evaluation operator, I want reporting to discover the frozen reference set, audit findings, programmatic findings, both match-decision sets, and live manifest, so that the correct evidence is assembled consistently. -33. As an evaluation operator, I want format failures read from the live manifest automatically, so that I do not duplicate run metadata in another document. -34. As an evaluation operator, I want missing or unresolved review artifacts reported with an actionable next command, so that the workflow explains how to proceed. -35. As an evaluation operator, I want JSON, CSV, and Markdown reports produced without a report bundle, so that no raw Python is required. -36. As a maintainer, I want obsolete authorization, plan-path, and report-bundle options removed immediately, so that the pre-proof-of-concept CLI has one supported workflow. -37. As a maintainer, I want the broader evaluation documentation updated to use live-run approval terminology, so that the design no longer implies that a digest authenticates an operator. -38. As a maintainer, I want the synthetic workflow to cover both audit and programmatic review decisions, so that the tested lifecycle matches the real reporting requirements. -39. As a project lead, I want these ergonomics changes kept separate from the actual Luna proof of concept, so that improving the CLI does not falsely mark the live evaluation complete. - -## Implementation Decisions - -- An evaluation run is the primary CLI concept joining one saved audit plan with its live evidence, reviewed match decisions, and reports. -- Dry audit planning automatically creates a unique run directory and prints its location for subsequent commands. -- The maximum audit-cost guardrail is mandatory during planning, validated as a positive decimal, and saved as part of the immutable plan. -- Live execution accepts the run directory rather than a separate plan path. It reads the saved guardrail and does not accept a replacement live value. -- Live execution continues to require the explicit live flag and an available OpenAI API key. -- The user-facing authorization digest, authorization option, one-use authorization receipt, authorized-plan duplicate field, and authorization-digest manifest field are removed. -- Internal content identities remain. Live execution revalidates the plan itself, frozen configuration, benchmark snapshot, approved eligibility decisions, prompt content, parser identity, pricing identity, and cost guardrail before any provider request. -- The complete billable-intent summary is rendered at live invocation before client construction. It includes model configuration, benchmark and reference identities, prompt names and count, estimated input tokens, retry policy, maximum audit cost, run directory, and approval mode. -- Interactive approval accepts only trimmed, case-insensitive `yes`. Any other response, end-of-input, or unavailable interactive terminal exits nonzero without constructing the provider client or making a request. -- Automatic approval is enabled only by an explicit `--auto-approve` option used together with live execution. It skips input but not summary rendering or any other gate. -- The live manifest records approval mode as `interactive` or `auto` plus an approval timestamp. It does not represent API-key presence as approval and does not claim a reviewer identity. -- Planning freezes a private copy of the approved reference set as run evidence. Scoring and reporting use that copy instead of the mutable current reference set. -- Live output paths are write-once. Existing live evidence causes an actionable failure that directs the operator to create a new evaluation run. -- Run-scoped match review distinguishes exactly two finding kinds: audit and programmatic. Each kind has a distinct workbook and validated match-decision artifact under a review partition keyed by run identifier. -- The review workflow accepts a run directory and finding kind, infers the conventional frozen reference set, findings input, review workbook, and match-decision output, and retains explicit overrides for exceptional workflows. -- Import mode continues to validate reviewer provenance, decision states, compatible evidence, required sub-defects, and one-to-one matching before writing decisions. -- Reporting accepts a run directory as its ordinary and only operator-facing input selector. It discovers all score inputs and obtains format failures directly from the live manifest. -- The report-bundle input and bundle-file format are removed from the CLI rather than retained as an advanced compatibility path. Tests and unconventional integrations can call the deterministic scorer interface directly. -- The obsolete plan-path option is removed because the plan has a conventional location within its evaluation run. -- Report formats, scoring rules, retry policy, prompt content, model configuration, normalization, matching semantics, and ranking behavior do not change. -- Documentation and the parent model-evaluation specification are revised to replace authorization-digest instructions with live-run approval and the run-oriented commands. -- Existing resolved implementation tickets remain resolved after their acceptance criteria and comments are made accurate. The operational Luna proof-of-concept ticket remains open until its private preparation, live execution, reviews, and final reports are actually completed. -- The domain glossary defines evaluation run and live-run approval. No architecture decision record is required because this pre-proof-of-concept interface change is straightforward to reverse and is not an architectural lock-in. - -## Testing Decisions - -- Tests assert externally visible behavior rather than private confirmation helpers, naming functions, or filesystem implementation order. -- The primary test seam is the evaluation CLI entry point exercised through a complete synthetic lifecycle in a temporary private workspace with a fake provider. -- The end-to-end CLI test plans a uniquely identified evaluation run, verifies the frozen reference set and saved cost limit, executes it with controlled approval, reviews audit findings, reviews programmatic findings, and generates all three reports without a report bundle. -- Focused CLI tests verify interactive acceptance, case and whitespace handling, rejection, end-of-input, unavailable terminal behavior, and automatic approval. -- Approval failure tests assert that no provider client is constructed, no request is made, and no live artifacts are created. -- Safety tests verify that the live flag, API key, saved positive cost guardrail, exact plan integrity, and frozen identities remain mandatory. -- Overwrite tests execute against occupied live destinations and assert that existing bytes remain unchanged and the command directs the operator to create a fresh evaluation run. -- Review tests prove that audit and programmatic decision outputs are distinct, run-scoped, validated, discoverable, and both required for final reporting. -- Reporting tests invoke only the run-directory workflow and verify automatic discovery of the frozen reference set, both findings collections, both match-decision sets, manifest metadata, and format failures. -- Parser tests verify that obsolete authorization, plan-path, and report-bundle options are rejected rather than silently ignored. -- Lower-level audit-executor tests remain for retry classification, cost accumulation, exhausted budgets, malformed successful responses, and immediate attempt persistence where those behaviors are clearer below the CLI seam. -- Existing synthetic evaluation CLI tests are the prior art. They will be reshaped around the run-oriented lifecycle rather than supplemented with a second parallel workflow. -- No test uses the private workbook, live Pristine content, a real API key, public network access, or billable provider calls. - -## Out of Scope - -- Performing the real Luna homepage proof of concept or checking off its operational acceptance criteria. -- Changing the GPT-5.6 Luna model, medium reasoning effort, endpoint, output limit, prompt order, filters, summaries, or retry policy. -- Changing eligibility classifications, match semantics, scoring rules, ranking metrics, report contents, or the illustrative-result limitation. -- Adding Astra-backed eligibility or match reviewers. -- Adding a web interface or public API for evaluation runs. -- Supporting report bundles as a hidden, deprecated, or advanced CLI feature. -- Authenticating the human who types `yes` or treating live-run approval as a cryptographic signature. -- Allowing approval to bypass API credentials, integrity checks, the cost guardrail, or overwrite protection. -- Migrating previously completed private evaluation artifacts; no real live proof-of-concept run exists yet. - -## Further Notes - -- The authorization digest was a confirmation checksum, not an identity proof. Point-of-use approval provides clearer operator intent while internal hashes continue to provide drift detection. -- A provider API key is a credential, not live-run approval. Its presence never triggers spending without the live flag and approval. -- A separate invocation, including one using automatic approval, constitutes a new live-run approval. A materially different configuration requires a newly planned evaluation run. -- The run directory is the operator's handle for live execution, review, and reporting even though private artifacts may remain partitioned by responsibility within the evaluation workspace. -- The existing private workbook and its SHA-256 remain unchanged by this feature. - -## Comments - -Implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by the -synthetic CLI lifecycle, focused approval and identity-drift tests, lower-level -retry and cost tests, compilation, and the full network-free pytest suite. The -operational Luna proof of concept remains open and no provider request was made. - - -## Haiku retarget clarification (2026-09-22) - -The Luna-specific target and credential details above are historical. They are -superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, -review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md b/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md deleted file mode 100644 index 90094d8..0000000 --- a/.scratch/model-evaluation/issues/01-explicit-audit-request-configuration.md +++ /dev/null @@ -1,25 +0,0 @@ -# 01: Make audit request configuration explicit - -**What to build:** Add a backward-compatible audit request seam that preserves current production behavior while allowing evaluation runs to specify model, reasoning effort, sampling omissions, output limit, and complete usage metadata. - -**Blocked by:** None (can start immediately). - -**Status:** resolved - -- [x] Existing Anthropic, OpenAI, and Gemini production callers retain their current effective defaults. -- [x] An OpenAI evaluation caller can explicitly request GPT-5.6 Luna with medium reasoning and no temperature parameter. -- [x] The request result exposes the provider metadata, token categories, stop reason, duration, and failure information required by evaluation runs. -- [x] Network-free tests verify legacy defaults and the Luna request shape through fakes. - -## Comments - -- Implemented in `c371979` with review fixes in `c395efa`. Verified by four - network-free audit-request tests and the production pipeline dry-run smoke - test. - - -## Haiku retarget clarification (2026-09-22) - -The Luna-specific target and credential details above are historical. They are -superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, -review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md b/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md deleted file mode 100644 index 8e14afc..0000000 --- a/.scratch/model-evaluation/issues/02-private-evaluation-workspace.md +++ /dev/null @@ -1,19 +0,0 @@ -# 02: Bootstrap a private, safe evaluation workspace - -**What to build:** Provide the CLI shell and private workspace conventions needed to prepare, run, review, and report evaluations without exposing employer data or accidentally spending money. - -**Blocked by:** None (can start immediately). - -**Status:** resolved - -- [x] The evaluation CLI exposes discoverable prepare, audit, review, and report workflows. -- [x] The source workbook is ignored by exact filename, and all generated references, reviews, snapshots, runs, and reports are ignored under one private workspace. -- [x] Missing private inputs produce actionable instructions rather than downloads or silent substitutions. -- [x] Dry-run behavior is the default even when an API key is present. -- [x] Excel support is added as a runtime dependency and pytest as a development dependency, with lock and generated dependency artifacts kept consistent. -- [x] Synthetic CLI tests prove workspace initialization and safe defaults without private data or network access. - -## Comments - -- Implemented in `aa77ee9` with safety fixes in `b99017f` and `e9485bf`. - Verified by the synthetic evaluation CLI test suite. diff --git a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md b/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md deleted file mode 100644 index 5f83e68..0000000 --- a/.scratch/model-evaluation/issues/03-freeze-pristine-benchmark-snapshot.md +++ /dev/null @@ -1,20 +0,0 @@ -# 03: Freeze a reproducible Pristine benchmark snapshot - -**What to build:** Add a preparation workflow that captures the raw Pristine homepage once and turns it into an immutable benchmark snapshot with enough provenance to reuse across model runs. - -**Blocked by:** 02: Bootstrap a private, safe evaluation workspace. - -**Status:** resolved - -- [x] The preparation workflow saves the exact raw HTML consumed by the existing audit pipeline. -- [x] Snapshot metadata records source URL, retrieval time, relevant HTTP metadata, and SHA-256. -- [x] Reusing a snapshot does not refetch or silently overwrite it. -- [x] The existing unrelated DAT Vision Aid fixture is rejected as a Pristine benchmark input. -- [x] Tests use local HTTP fakes and verify content identity, provenance, hashing, and overwrite protection. - -## Comments - -- Implemented in `0bd719c` with review hardening in `fc630cb`. Verified by - local-HTTP snapshot tests covering byte identity, provenance, SHA-256, - no-refetch reuse, UTF-8 compatibility, atomic publication, fixture rejection, - and overwrite protection; the full suite passes with 17 tests. diff --git a/.scratch/model-evaluation/issues/04-homepage-reference-set.md b/.scratch/model-evaluation/issues/04-homepage-reference-set.md deleted file mode 100644 index f3f6ff0..0000000 --- a/.scratch/model-evaluation/issues/04-homepage-reference-set.md +++ /dev/null @@ -1,22 +0,0 @@ -# 04: Generate and validate the homepage reference set - -**What to build:** Transform the mandated workbook's Home and applicable Global rows into a traceable, human-reviewed reference set and generate the private Eligibility review needed to reach the no-spend approval gate. - -**Blocked by:** 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot. - -**Status:** resolved - -- [x] Workbook absence and SHA-256 drift fail with clear acquisition or validation guidance. -- [x] Imported records retain workbook filename, sheet, source row, raw evidence, canonical URL, page scope, and WCAG evidence without semantic rewriting. -- [x] The reference set includes every homepage and applicable Global row exactly once. -- [x] Eligibility is limited to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with rationale and approval state required. -- [x] Global rows are represented as global claims with homepage evidence rather than multiplied across pages. -- [x] A private Eligibility workbook presents source evidence, proposed classification, rationale, approval, reviewer identity, confidence, timestamp, and notes. -- [x] Human-reviewed decisions round-trip into a validated, versioned reference set through the `EligibilityReviewer` interface. -- [x] Synthetic workbook tests cover provenance, filtering, review import, invalid decisions, and checksum validation. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Verified by the synthetic reference-set and CLI tests within the -55-test passing suite. diff --git a/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md b/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md deleted file mode 100644 index e4023c3..0000000 --- a/.scratch/model-evaluation/issues/05-review-and-score-synthetic-comparison.md +++ /dev/null @@ -1,22 +0,0 @@ -# 05: Review and score a synthetic comparison - -**What to build:** Demonstrate the complete deterministic review-to-report path with synthetic reference defects and findings, establishing the scoring seam before connecting real audit outputs. - -**Blocked by:** 04: Generate and validate the homepage reference set. - -**Status:** resolved - -- [x] The `MatchReviewer` interface accepts human-reviewed decisions without coupling the scorer to Excel or future model adapters. -- [x] Match decisions support accepted, rejected, and needs-review states with reviewer provenance, rationale, confidence, and timestamp. -- [x] Validation enforces one-to-one matches and rejects conflicting accepted decisions. -- [x] Grouped rows receive one point when their grouped defect is matched, while rows with distinct required sub-defects require all parts. -- [x] The deterministic scorer calculates workbook-row recall, programmatic coverage, combined workbook coverage, unmatched findings, and parse failures from approved inputs. -- [x] Estimated audit cost is the sole ranking tie-breaker; latency, token usage, and evaluation cost remain informational. -- [x] JSON, CSV, and Markdown reports derive from one score object and agree on every total and row disposition. -- [x] Repeated scoring of identical inputs produces byte-equivalent structured results apart from deliberately excluded presentation timestamps. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Verified by deterministic scoring, matching, ranking, and -cross-format report tests within the 55-test passing suite. diff --git a/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md b/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md deleted file mode 100644 index ebc0310..0000000 --- a/.scratch/model-evaluation/issues/06-programmatic-workbook-coverage.md +++ /dev/null @@ -1,21 +0,0 @@ -# 06: Measure programmatic workbook coverage - -**What to build:** Run the existing deterministic accessibility checks against the benchmark snapshot and show which workbook rows they actually cover, separately from model efficacy. - -**Blocked by:** 04: Generate and validate the homepage reference set; 05: Review and score a synthetic comparison. - -**Status:** resolved - -- [x] Existing semantic, form, and non-text programmatic checks run against the frozen snapshot without duplicated audit logic. -- [x] Programmatic findings are normalized with stable page and location evidence while retaining their raw source. -- [x] Reference matches are reviewable and use the same accepted-decision validation as other matches. -- [x] A `programmatic` eligibility classification does not automatically award credit when the checker misses the row. -- [x] Reports identify covered programmatic rows, uncovered checker responsibilities, unavailable evidence, and ambiguity. -- [x] Programmatic results contribute to combined workbook coverage but never to model ranking. -- [x] Tests exercise the complete programmatic path with synthetic HTML and approved match decisions. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Verified with synthetic HTML, approved match decisions, and the -production pipeline dry run. diff --git a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md b/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md deleted file mode 100644 index 29de7a3..0000000 --- a/.scratch/model-evaluation/issues/07-safe-luna-dry-run.md +++ /dev/null @@ -1,30 +0,0 @@ -# 07: Prepare a safe Luna dry run - -**What to build:** Produce a no-cost Luna audit plan against the approved benchmark configuration, with complete provenance and an explicit summary suitable for point-of-use live-run approval. - -**Blocked by:** 01: Make audit request configuration explicit; 02: Bootstrap a private, safe evaluation workspace; 03: Freeze a reproducible Pristine benchmark snapshot; 04: Generate and validate the homepage reference set. - -**Status:** resolved - -- [x] The dry run uses the frozen prompts, filters, prompt order, disabled summaries, sequential execution, and existing output limit. -- [x] The planned configuration records GPT-5.6 Luna, medium reasoning, Chat Completions, and omitted temperature. -- [x] Prompt payloads, request count, estimated input tokens, prompt hashes, parser hashes, and repository state are saved without making a provider call. -- [x] The manifest records workbook, snapshot, reference-set, benchmark, configuration, and pricing identities. -- [x] Planning creates a unique evaluation run, freezes the approved reference set, and requires a positive maximum audit-cost limit. -- [x] The command refuses live execution when the live flag, API key, saved positive cost guardrail, exact planned evidence, or live-run approval is absent. -- [x] Tests prove that an environment key alone cannot trigger spending. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. The run-oriented planning workflow was -implemented in `2448042` and review-hardened in `6a1e6c0`. Verified by unique -run, frozen-evidence, positive-cost, identity-drift, and no-spend tests; no -provider call is made by the test suite. - - -## Haiku retarget clarification (2026-09-22) - -The Luna-specific target and credential details above are historical. They are -superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, -review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/08-authorized-live-audit.md b/.scratch/model-evaluation/issues/08-authorized-live-audit.md deleted file mode 100644 index 22800a6..0000000 --- a/.scratch/model-evaluation/issues/08-authorized-live-audit.md +++ /dev/null @@ -1,33 +0,0 @@ -# 08: Execute and record an approved live audit - -**What to build:** Add the guarded live-audit path that executes a point-of-use approved Luna run, preserves every raw outcome, and accounts for reliability, usage, cost, and latency. - -**Blocked by:** 01: Make audit request configuration explicit; 07: Prepare a safe Luna dry run. - -**Status:** resolved - -- [x] Live execution requires the explicit live flag, API key, saved maximum audit-cost limit, exact summarized benchmark configuration, and interactive or automatic approval. -- [x] Audit prompts run sequentially with medium reasoning, no temperature, disabled summaries, and the existing output limit. -- [x] At most two additional attempts occur for timeouts, rate limits, and server errors using bounded exponential backoff. -- [x] Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. -- [x] Every attempt records request identity, response metadata, usage, duration, stop reason, and failure classification without exposing secrets. -- [x] Billed retry usage contributes to audit cost, and the runner stops before starting another request once the cost guardrail is exhausted. -- [x] Exhausted transient failures mark the run incomplete and unranked; malformed successful responses remain recorded as model-format failures. -- [x] The live manifest records approval mode and timestamp without claiming operator identity. -- [x] Existing live evidence is never overwritten; another execution requires a newly planned evaluation run. -- [x] Fake-provider tests exercise success, retry, budget, malformed response, and incomplete-run behavior without spending money. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Point-of-use approval and write-once run -execution were implemented in `2448042` and copy/rerun isolation was hardened -in `6a1e6c0`. Fake-provider tests verify approval, retry, checkpoint, budget, -malformed-response, and incomplete-run policies without spending. - - -## Haiku retarget clarification (2026-09-22) - -The Luna-specific target and credential details above are historical. They are -superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, -review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md b/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md deleted file mode 100644 index ac0b0a5..0000000 --- a/.scratch/model-evaluation/issues/09-normalize-raw-audit-findings.md +++ /dev/null @@ -1,20 +0,0 @@ -# 09: Normalize every raw audit prompt response - -**What to build:** Convert saved raw audit responses into lossless canonical audit findings without relying on the production CSV, suppression rules, or model-based deduplication. - -**Blocked by:** 08: Execute and record an approved live audit. - -**Status:** resolved - -- [x] Every active non-summary prompt has a fixed normalizer, including categories omitted by the production CSV. -- [x] Canonical audit findings retain run and model identity, prompt, checklist, page, problem family and statement, location evidence, WCAG evidence, raw source, and parse status. -- [x] Valid object, array, fenced JSON, empty, malformed, and unknown prompt responses produce explicit deterministic outcomes. -- [x] Successful malformed responses produce no usable findings and remain visible as format failures. -- [x] Normalization performs no model or network call and applies no heuristic suppression or deduplication. -- [x] Synthetic fixtures cover every prompt normalizer without copying private or live provider data. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Synthetic normalization tests cover the complete prompt registry -and all specified parse outcomes. diff --git a/.scratch/model-evaluation/issues/10-review-finding-matches.md b/.scratch/model-evaluation/issues/10-review-finding-matches.md deleted file mode 100644 index fd472f9..0000000 --- a/.scratch/model-evaluation/issues/10-review-finding-matches.md +++ /dev/null @@ -1,25 +0,0 @@ -# 10: Generate and validate finding matches - -**What to build:** Turn canonical audit findings and approved reference defects into an efficient, evidence-rich human match review whose accepted decisions can be scored deterministically. - -**Blocked by:** 04: Generate and validate the homepage reference set; 09: Normalize every raw audit prompt response. - -**Status:** resolved - -- [x] Candidate generation removes only hard incompatibilities such as different pages or incompatible evidence categories. -- [x] Candidate generation never awards credit or makes a semantic match decision. -- [x] A private Matches workbook presents reference evidence, audit evidence, proposed disposition, rationale, confidence, approval, reviewer identity, timestamp, and notes side by side. -- [x] An accepted match requires compatible page or scope, the same underlying accessibility failure, and compatible location, element, or element group. -- [x] WCAG identifiers and remediation wording remain supporting evidence rather than exact-match requirements. -- [x] Approved decisions round-trip through the `MatchReviewer` interface and enforce one-to-one matching. -- [x] Conflicts, unresolved decisions, incomplete grouped evidence, and missing required sub-defects fail clearly rather than being resolved silently. -- [x] Audit and programmatic findings use distinct run-scoped workbooks and validated decision artifacts. -- [x] The CLI infers conventional review paths from the evaluation run and finding kind while retaining explicit overrides. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. Run-scoped audit and programmatic review -discovery was implemented in `2448042` and historical-run loading was hardened -in `6a1e6c0`. Candidate generation, workbook round-trip, evidence validation, -and conflict tests pass in the synthetic suite. diff --git a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md b/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md deleted file mode 100644 index ffab7b0..0000000 --- a/.scratch/model-evaluation/issues/11-verify-complete-evaluation-workflow.md +++ /dev/null @@ -1,24 +0,0 @@ -# 11: Verify the complete evaluation workflow - -**What to build:** Connect preparation, audit, review, scoring, and reporting into one coherent workflow and prove the entire lifecycle with synthetic provider data before any real model run. - -**Blocked by:** 05: Review and score a synthetic comparison; 06: Measure programmatic workbook coverage; 08: Execute and record an approved live audit; 10: Generate and validate finding matches. - -**Status:** resolved - -- [x] The CLI completes prepare, audit, review, and report workflows against a temporary private workspace using synthetic inputs. -- [x] Ordinary comparisons reject mismatched snapshots, reference sets, prompts, parsers, eligibility decisions, or benchmark configurations. -- [x] Run manifests contain content identity, code state, request configuration, retry policy, usage, pricing, timestamps, and run identity. -- [x] Audit, evaluation, and total experiment costs remain separate, and only audit cost affects equal-recall ranking. -- [x] End-to-end and per-request latency are reported but never ranked. -- [x] JSON, CSV, and Markdown reports expose every in-scope workbook row and agree on scoring and coverage. -- [x] CI runs the pytest suite without private workbook data, public network access, provider credentials, or spending. -- [x] Documentation explains the private-data workflow, evaluation-run planning, point-of-use live-run approval, run-scoped reviews, discovered reporting inputs, and illustrative limits of the homepage benchmark. - -## Comments - -Implemented in `472500c`, hardened in `8bfe6af`, style-reviewed in `1cb479a`, -and identity-verified in `2129616`. The run-oriented synthetic lifecycle and -documentation were implemented in `2448042` and review-hardened in `6a1e6c0`. -The suite proves both review kinds, discovered reporting, approval failure -paths, immutable identities, and write-once execution without provider access. diff --git a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md b/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md deleted file mode 100644 index 1a5d7cc..0000000 --- a/.scratch/model-evaluation/issues/12-run-luna-homepage-poc.md +++ /dev/null @@ -1,29 +0,0 @@ -# 12: Run the gated Haiku homepage proof of concept - -**What to build:** Execute the approved operational workflow on the private Pristine benchmark and produce the first illustrative Haiku evaluation report without bypassing any human or spending gate. - -**Blocked by:** 11: Verify the complete evaluation workflow. - -**Status:** ready-for-agent - -- [ ] The private workbook checksum, snapshot, reference set, and eligibility classifications are reviewed and approved before the audit proceeds. -- [ ] A no-cost evaluation-run summary presents the exact Haiku configuration, request set, estimated usage, and planned cost limit. -- [ ] The live run begins only after point-of-use approval covering the displayed configuration and saved cost guardrail. -- [ ] The approved Haiku run preserves all raw responses, attempts, usage, cost, latency, failures, and provenance. -- [ ] Canonical findings and candidate matches are generated without production CSV filtering or model-based deduplication. -- [ ] Ambiguous match decisions receive human approval before scoring. -- [ ] The final reports show Haiku workbook-row recall, programmatic coverage, combined workbook coverage, estimated audit cost, tokens, latency, unmatched findings, and format failures. -- [ ] The result is labeled illustrative and makes no small-versus-large model claim from the homepage's limited eligible denominator. - -## Comments - -The run-oriented CLI ergonomics were implemented in `2448042` and hardened in -`6a1e6c0`. This operational ticket remains open: no private preparation, live -Luna request, human match review, or final proof-of-concept report was performed. - - -2026-09-22: The active target is now pinned `claude-haiku-4-5-20251001`, -with 16,000 thinking tokens and 24,192 total output tokens via Anthropic -Messages. See `../../anthropic-evaluation-retarget/spec.md`. This operational -ticket remains open: the retarget ends at a dry-run plan, with no live approval -or paid request. Historical Luna attempts remain immutable. diff --git a/.scratch/model-evaluation/progress-report.md b/.scratch/model-evaluation/progress-report.md deleted file mode 100644 index 3c4376c..0000000 --- a/.scratch/model-evaluation/progress-report.md +++ /dev/null @@ -1,450 +0,0 @@ -# Model evaluation suite progress report - -Updated: September 20, 2026 - -## Term used in this report - -**Reference workbook** means the authorized Excel file containing the findings -from the prior human accessibility audit. Its individual tabs are worksheets, -and their rows supply the source evidence for the benchmark's reference -defects. The generated Eligibility and Matches Excel files are review -workbooks, not the reference workbook. - -## Why the evaluation suite is separate - -- I built a separate evaluation suite because the production CSV is a cleaned - customer deliverable rather than a complete record of model performance. - -- The CSV has no normalizers for six registered non-summary prompt categories: - table semantics, placeholder-only labels, form-group labels, form - instructions, complex image descriptions, and media captions. - -- The CSV also skips failed or malformed responses, combines model and - programmatic findings, applies false-positive filters, and can use another - LLM for deduplication. - -- Scoring that output would measure the report-processing pipeline along with - the audit model. - -- The evaluation suite instead normalizes every raw prompt response, preserves - failures and provenance, separates programmatic coverage, and scores only - approved matches against eligible workbook rows. - -- I reused the production pipeline for prompt generation and provider requests - so the evaluation still tests the same audit behavior used by the - application. - -## CLI command order - -Run the following commands in order. Complete the Eligibility workbook before -command 2 and each generated Matches workbook before its following -`--import-matches` command. `RUN_DIR` is the evaluation-run path printed by -command 3. - -```bash -# 1. Build the benchmark from the human-audit reference workbook. -# This imports the reference defects and freezes the homepage HTML. -# --homepage-url identifies which workbook page rows belong to the -# benchmark. It selects the Home and applicable Global reference defects -# but does not fetch the webpage. -# --snapshot-url identifies where to download the HTML that models will -# inspect. It records the captured content and its provenance. -# Both values are the same for this proof of concept because the workbook -# homepage and the captured benchmark page are both pristineai.com. -uv run visionaid-evaluate prepare \ - --workbook-sha256 "$WORKBOOK_SHA256" \ - --homepage-url https://pristineai.com/ \ - --snapshot-url https://pristineai.com/ - -# Manually complete .model-evaluation/reviews/eligibility.xlsx. -# Classify who is responsible for each reference defect before scoring begins. - -# 2. Validate the completed eligibility decisions and create the approved -# reference set used by every evaluation run. -uv run visionaid-evaluate review \ - --eligibility-workbook .model-evaluation/reviews/eligibility.xlsx - -# 3. Plan the model evaluation without making provider requests. -# This freezes the prompts, model settings, evidence identities, and cost -# guardrail. Save the evaluation-run path printed by this command. -uv run visionaid-evaluate audit --max-audit-cost-usd 1.00 - -RUN_DIR=".model-evaluation/runs/REPLACE_WITH_PRINTED_RUN_ID" - -# 4. Generate a workbook that compares deterministic checker findings with -# the human-audit reference defects. This review measures checker coverage, -# not model performance. -uv run visionaid-evaluate review \ - --run-dir "$RUN_DIR" \ - --kind programmatic - -# Manually complete the generated programmatic Matches workbook. - -# 5. Validate and save the programmatic match decisions. -uv run visionaid-evaluate review \ - --run-dir "$RUN_DIR" \ - --kind programmatic \ - --import-matches - -# 6. Display the saved billable intent, request approval, and run the model. -# The command preserves raw responses, usage, failures, and canonical audit -# findings. It cannot overwrite evidence from an earlier live attempt. -uv run visionaid-evaluate audit \ - --live \ - --run-dir "$RUN_DIR" - -# 7. Generate a workbook that compares the model's findings with the eligible -# human-audit reference defects. These matches determine model recall. -uv run visionaid-evaluate review \ - --run-dir "$RUN_DIR" \ - --kind audit - -# Manually complete the generated audit Matches workbook. - -# 8. Validate and save the audit match decisions used for model scoring. -uv run visionaid-evaluate review \ - --run-dir "$RUN_DIR" \ - --kind audit \ - --import-matches - -# 9. Verify that the live evidence and both reviews belong to this run, then -# calculate the score and write JSON, CSV, and Markdown reports. -uv run visionaid-evaluate report --run-dir "$RUN_DIR" -``` - -## CLI workflow in detail - -The four commands describe stages of one evaluation rather than four unrelated -utilities. Each stage produces evidence that the next stage verifies and uses. - -### `prepare` - -- I added `prepare` to establish the benchmark before any model is evaluated. - It requires an authorized local workbook rather than downloading or - substituting private source data. - -- It creates the ignored `.model-evaluation` workspace so private inputs and - derived evidence stay outside version control. - -- When supplied with the approved workbook checksum and homepage URL, it - verifies the workbook, imports every applicable Home and Global row, and - preserves the original sheet, row, wording, location, WCAG evidence, and - recommendation. - -- It writes `references/imported.json` as the unapproved reference inventory - and `reviews/eligibility.xlsx` as the human-readable classification - workbook. Importing a row does not make it scorable until its eligibility - decision is reviewed and accepted. - -- When supplied with a snapshot URL, it captures the exact HTML that the audit - pipeline will consume and records the source URL, capture time, HTTP - metadata, and SHA-256. - -- It refuses to overwrite or silently recapture an existing snapshot because - changing page content between model runs would invalidate the comparison. - -- `prepare` makes no provider request and incurs no model cost. - -### `audit` - -- I made `audit` dry by default because possessing an API key should never be - enough to spend money. - -- In planning mode, it requires an accepted reference set, the frozen - snapshot, and a positive maximum audit cost. It then runs the production - pipeline without provider calls to determine the exact prompts and payloads - that would be sent. - -- Planning creates a unique evaluation-run directory and freezes the approved - reference set, prompt text, payloads, prompt hashes, parser identity, model - configuration, pricing identity, retry policy, repository state, estimated - input usage, and cost guardrail. - -- It prints the model, endpoint, reasoning effort, benchmark identity, prompt - names, request count, retry policy, estimated input usage, maximum cost, and - output destination so billable intent is visible before execution. - -- Live mode is a separate invocation that requires `--live`, the planned run - directory, an API key, and interactive or explicit automatic approval. It - revalidates the saved evidence before creating the provider client. - -- Live execution sends prompts sequentially, records every attempt, applies - bounded retries only to transient failures, tracks billed usage against the - saved cost limit, and stops before another request could exceed that limit. - -- It preserves raw responses and provider metadata, then normalizes successful - responses into canonical audit findings. Authentication errors, malformed - output, exhausted retries, and incomplete runs remain recorded instead of - disappearing from the result. - -- A live destination is write-once. Repeating an experiment requires a newly - planned run so evidence from separate approvals cannot be mixed. - -### `review` - -- I added `review` because neither benchmark eligibility nor semantic finding - matches can be decided safely from string similarity alone. - -- The first review occurs after preparation. A reviewer classifies every - imported reference in `eligibility.xlsx`, records a rationale, decision, - identity, confidence, and timestamp, then imports the completed workbook as - `references/approved.json`. - -- The second review occurs per evaluation run. The command creates separate - match workbooks for `audit` findings and `programmatic` findings so model - performance cannot be confused with deterministic checker coverage. - -- Each match workbook presents reference evidence and finding evidence side by - side. The reviewer decides whether they describe the same underlying - accessibility failure at a compatible page, location, element, or element - group. - -- Importing a completed match workbook validates decision states, reviewer - provenance, evidence compatibility, required sub-defects, and one-to-one - matching before it writes the decision JSON used for scoring. - -- Candidate generation and workbook export never award points. The scorer - receives only accepted, validated decisions. - -### `report` - -- I added `report` as the final deterministic stage so presentation cannot - change the approved score. - -- It accepts an evaluation-run directory and discovers the frozen reference - set, canonical audit findings, canonical programmatic findings, both sets of - approved match decisions, and the live manifest. - -- It verifies that those artifacts belong to the same run and refuses to - report when live execution, normalization, or either review is missing. - -- It calculates workbook-row recall, programmatic coverage, combined workbook - coverage, estimated audit cost, tokens, latency, unmatched findings, and - format failures from the verified evidence. - -- It writes JSON, CSV, and Markdown from one score object so all three formats - agree on totals and individual workbook-row dispositions. - -- `report` does not call a model or reinterpret review decisions. Running it - again against unchanged evidence produces the same scoring result. - -## Shared model request layer - -- I created `AuditRequestClient` so model evaluations exercise the same - provider request code used in production. This prevents request behavior from - drifting between the application and the evaluation suite. - -- I added `AuditRequestConfig` to control and record the model, reasoning - effort, temperature, and maximum output tokens for each run. - -- I added configurable reasoning effort so OpenAI reasoning models can be - compared at a known setting. - -- I allowed temperature to be omitted because some models reject the parameter - rather than accepting a value. - -- I added model capability checks to prevent unsupported parameters from - causing failed requests. - -- I preserved production's existing temperature and output-token defaults so - moving production to the shared client would not change current audit - behavior. - -- I added automatic routing for Anthropic, OpenAI, and Gemini so callers do not - need provider-specific request logic. - -- I recorded requested and resolved configuration so each evaluation shows - what settings the provider received. - -- I added detailed token usage for cost calculations, including cached and - reasoning tokens when providers report them. - -- I added provider response details, stop reasons, duration, and structured - failures so incomplete runs can be diagnosed without repeating billable - requests. - -- I added provider-client injection so tests can inspect exact request - parameters without making live API calls. - -## Private inputs and benchmark evidence - -- I created an ignored `.model-evaluation` workspace because the source - workbook, derived references, captured HTML, reviews, raw responses, and - reports contain private project data. - -- I added workbook checksum validation because a changed or substituted - workbook would silently change the benchmark. - -- I preserved each reference defect's original workbook sheet, row, wording, - location, WCAG evidence, and recommendation so every score can be traced back - to its source. - -- I added an immutable homepage snapshot with its URL, capture time, HTTP - metadata, and SHA-256 because every model must receive the same page content. - -## Eligibility and scoring - -- I added eligibility review because the model should only lose points for - defects it had enough evidence and responsibility to detect. - -- I defined four eligibility classifications: `llm_eligible`, `programmatic`, - `unavailable_evidence`, and `ambiguous`. - -- `llm_eligible` means the audit model receives enough evidence in its prompt - and is responsible for making the required accessibility judgment. Only - accepted references in this class enter the model-recall denominator. This - classification does not assume the model will find the defect. - -- `programmatic` means the defect belongs to a deterministic checker rather - than the audit model. The row counts toward programmatic and combined - coverage only when an actual programmatic finding is reviewed and matched; - the classification alone does not award credit. - -- `unavailable_evidence` means the current audit pipeline does not receive the - visual, styling, interaction, cross-page, or other evidence needed to judge - the workbook claim. It identifies a benchmark limitation, not a model miss - and not a claim that the source defect is invalid. - -- `ambiguous` means the workbook claim, expected scope, or required sub-defects - cannot yet be interpreted confidently enough for fair assignment or - scoring. It describes uncertainty in the reference, not uncertainty in a - model response. - -- An accepted `ambiguous` classification is a completed review decision. It is - different from an eligibility decision that still has a `needs_review` - state. - -- I kept `unavailable_evidence` and `ambiguous` references visible in reports - but excluded them from both the model and programmatic denominators. This - prevents missing evidence or unresolved benchmark meaning from helping or - hurting a model. - -- I separated programmatic coverage from model recall because deterministic - checker results should not improve or reduce a model's ranking. - -- I added `EligibilityReviewer` and `MatchReviewer` interfaces so human - workbook review and a future Astra reviewer can produce the same validated - decisions without changing scoring or reporting. - -- I kept the scorer separate from review so it calculates results only from - approved decisions and produces the same result from the same inputs. - -- I made workbook-row recall over accepted `llm_eligible` defects the primary - model metric because the mandated workbook is the benchmark source. - -- I made estimated audit cost the only ranking tie-breaker. Token usage, - latency, unmatched findings, malformed output, and programmatic coverage - remain visible but do not alter rank. - -## Run safety and reproducibility - -- I added evaluation-run planning because the model configuration, prompt set, - benchmark, reference set, parser versions, pricing, retry policy, and cost - limit must be fixed before a billable request begins. - -- I added a positive cost guardrail, explicit `--live` flag, API-key - requirement, and point-of-use approval because credentials alone must never - trigger spending. - -- I made live-run output immutable because retrying into the same directory - would mix evidence from separate executions. - -- I added bounded retries for timeouts, rate limits, and server failures - because those failures may be temporary. - -- I disabled retries for authentication errors, invalid requests, and - malformed successful responses because repeating them would spend money - without addressing the cause. - -## Normalization, review, and reporting - -- I added canonical finding normalization for every registered non-summary - prompt because provider responses use different JSON shapes that cannot be - compared directly. - -- I preserved malformed responses and parsing failures instead of dropping - them because output-format reliability is part of the evaluation evidence. - -- I added candidate-match workbooks because semantic accessibility findings - cannot be matched safely by WCAG number or text similarity alone. - -- I enforced one-to-one approved matches because allowing one finding to - satisfy several unrelated workbook rows would inflate recall. - -- I added JSON, CSV, and Markdown reports from one score object so every format - agrees on totals, row dispositions, coverage, cost, and failures. - -## Verification - -- I added synthetic tests for the complete lifecycle so CI can verify the - suite without the private workbook, public network access, provider - credentials, or billable requests. - -- The current test suite passes all 76 tests. - -- A production pipeline dry run also passed without provider calls. It produced - 14 programmatic findings and generated 10 model prompts from the committed - HTML fixture. - -## Current proof-of-concept status - -- The evaluation framework is complete. Eleven implementation tickets are - resolved, and the remaining ticket covers the real Luna homepage proof of - concept. - -- The private homepage benchmark contains 16 reviewed reference defects: five - programmatic, three `llm_eligible`, six unavailable to the current pipeline, - and two ambiguous. - -- The three `llm_eligible` defects form the current model-recall denominator. - Unavailable and ambiguous defects remain reported without affecting the - score. - -- The frozen Pristine homepage snapshot and its provenance are saved in the - private workspace. - -- The benchmark dry run generated 29 programmatic findings and eight active - model prompts. - -- Five programmatic workbook rows have accepted one-to-one matches. The - remaining programmatic candidates were reviewed and rejected. - -- The first Luna run was planned for eight sequential requests with 5,776 - estimated input tokens, medium reasoning, no temperature parameter, and a - $1.00 cost limit. - -- The provider rejected the first live request because the API key was invalid. - The runner stopped immediately, used zero tokens, incurred no cost, and - marked the run incomplete and unrankable. - -- The failed run remains preserved rather than overwritten. - -- A fresh replacement run has been planned, and its five programmatic match - decisions have been validated. - -- The replacement run has not made a live provider request. It is waiting for - a valid OpenAI Platform API key. - -- There is no Luna efficacy score yet. Reporting a score before the replacement - live run and audit-match review would misrepresent the current state. - -## Remaining work - -- Replace the rejected credential with a valid OpenAI Platform API key. - -- Execute the prepared replacement Luna run. - -- Generate and review the audit-finding match workbook. - -- Import the approved audit match decisions. - -- Generate the final JSON, CSV, and Markdown evaluation reports. - - -## Haiku retarget (2026-09-22) - -The Luna replacement plan is superseded by a fresh, unexecuted Haiku plan. -Current configuration: Anthropic Messages, `claude-haiku-4-5-20251001`, -16,000 thinking tokens, 24,192 total output tokens, no temperature. See the -private operator log for the new run identity. No paid run is authorized or -performed by this retarget; operational ticket 12 remains open. diff --git a/.scratch/model-evaluation/spec.md b/.scratch/model-evaluation/spec.md deleted file mode 100644 index ab14ea3..0000000 --- a/.scratch/model-evaluation/spec.md +++ /dev/null @@ -1,184 +0,0 @@ -Status: ready-for-agent - -## Problem Statement - -The project needs a trustworthy way to compare how effectively different AI models perform the repository's accessibility-auditing role, especially whether smaller, less expensive models find nearly as many known accessibility problems as larger models. The mandated source of reference problems is the Pristine Accessibility Defect Report workbook, whose prose, locations, and recommendations are not standardized enough for direct automated scoring. The current audit pipeline also combines deterministic checks with AI judgment, filters some programmatically detectable defects out of model inputs, emits heterogeneous prompt responses, and produces a final CSV that drops some active prompt categories and applies extra filtering. Without a separate evaluation framework, a model comparison would conflate model performance with deterministic findings, missing evidence, post-processing, page drift, and subjective matching. - -The immediate goal is not to prove that a small model is equivalent to a large model. It is to prove that the evaluation method can import the required workbook, identify which reference defects the AI model is responsible for finding, normalize raw audit findings without losing data, support reviewable one-to-one matching, calculate an exact repeatable score, and report efficacy and cost transparently. The first proof of concept will use GPT-5.6 Luna at medium reasoning against a frozen Pristine homepage snapshot. - -## Solution - -Build a private, CLI-only model-evaluation framework around the existing audit pipeline. The framework will preserve the workbook as the required source of truth while transforming its homepage rows into human-reviewed reference defects with explicit eligibility classifications. It will freeze and hash a Pristine homepage benchmark snapshot, run the existing programmatic and AI audit paths without changing prompts, normalize raw prompt responses into canonical audit findings, generate reviewable candidate matches, and calculate workbook-row recall from approved match decisions. - -The deterministic scorer will remain separate from review. Human-reviewed files will initially implement the eligibility and match reviewer interfaces; future Astra-backed adapters can produce the same decisions without changing import, scoring, or reporting. Models will be ranked first by workbook-row recall over approved LLM-eligible rows and, only when recall is equal, by estimated audit cost. Programmatic coverage, combined workbook coverage, latency, token usage, malformed output, and unmatched findings will remain visible but will not alter model rank. - -Private inputs and generated artifacts will stay outside version control. The committed repository will contain only the framework, schemas, documentation, versioned public pricing configuration, and synthetic tests. Live API execution will be impossible without an explicit live flag, a planned positive cost limit, an available key, and point-of-use live-run approval after the complete execution summary. - -## User Stories - -1. As an accessibility-audit researcher, I want to compare models against the mandated defect workbook, so that results satisfy the project's external reference-data requirement. -2. As an accessibility-audit researcher, I want the first proof of concept to validate the measurement method, so that a weak or strong Luna result is not confused with evaluator quality. -3. As a project lead, I want every scored result to trace back to a workbook source row, so that I can explain the benchmark to stakeholders. -4. As a benchmark curator, I want the workbook checksum recorded, so that an updated or substituted workbook cannot silently change results. -5. As a contributor, I want a clear error when the private workbook is absent, so that I know to obtain it from an authorized project source. -6. As a contributor, I want to override the default workbook location explicitly, so that local workflows are not tied to one machine layout. -7. As a data owner, I want the workbook excluded from version control, so that employer-provided information is not published without approval. -8. As a data owner, I want workbook-derived references, reviews, snapshots, runs, and reports excluded from version control, so that derived private information is protected too. -9. As a benchmark curator, I want each workbook row imported without semantic rewriting, so that the original evidence remains intact. -10. As a benchmark curator, I want each reference defect to retain its source sheet and row, so that normalization is auditable. -11. As a benchmark curator, I want each homepage and applicable Global row classified, so that no workbook row silently disappears. -12. As a benchmark curator, I want the allowed eligibility classifications validated, so that references consistently distinguish AI responsibility, programmatic responsibility, unavailable evidence, and ambiguity. -13. As an evaluator, I want LLM-eligible reference defects clearly separated from programmatic defects, so that models are scored only on work assigned to them. -14. As a project lead, I want unavailable-evidence rows reported with reasons, so that pipeline blind spots remain visible. -15. As a project lead, I want ambiguous rows excluded from the scoring denominator until approved, so that unresolved interpretation does not arbitrarily help or hurt a model. -16. As a comparison reviewer, I want eligibility proposals presented with workbook and pipeline evidence, so that I can approve classifications efficiently. -17. As a comparison reviewer, I want a private Excel review workbook, so that multiline evidence and decisions can be reviewed side by side. -18. As a comparison reviewer, I want accepted, rejected, and needs-review decisions, so that uncertainty is explicit. -19. As a comparison reviewer, I want reviewer identity, rationale, confidence, and timestamp retained, so that decisions have provenance. -20. As a future automation maintainer, I want human and model reviewers to satisfy the same interfaces, so that Astra can replace tedious review without changing the scorer. -21. As a future automation maintainer, I want eligibility review and match review to be separate interfaces, so that benchmark curation and per-run adjudication can evolve independently. -22. As a data owner, I want future Astra review to use the authorized API project without publishing inputs, so that automation does not weaken data privacy. -23. As an evaluator, I want a benchmark snapshot captured once from the Pristine homepage, so that every model sees identical page content. -24. As an evaluator, I want the snapshot URL, capture time, HTTP metadata, and SHA-256 recorded, so that the tested page state is identifiable. -26. As an evaluator, I want Global rows scored only for their homepage manifestation in this proof of concept, so that homepage evidence is not inflated into a site-wide claim. -27. As an evaluator, I want the existing programmatic audit run against the benchmark snapshot, so that deterministic workbook coverage and checker gaps are measured. -28. As an evaluator, I want programmatic findings excluded from the AI model score, so that model-independent results do not inflate similarity between models. -29. As a project lead, I want programmatic coverage reported separately, so that the value and gaps of deterministic checks are visible. -30. As a project lead, I want combined workbook coverage reported separately, so that I can see what the complete two-pass pipeline catches. -31. As an evaluator, I want the current prompts, filters, prompt order, and summary setting frozen, so that the proof of concept measures the current audit role rather than a redesigned prompt system. -32. As an evaluator, I want GPT-5.6 Luna called with medium reasoning and without inherited temperature sampling, so that the requested model configuration is explicit. -33. As an evaluator, I want all raw prompt responses saved, so that normalization and scoring can be audited later. -34. As an evaluator, I want every active prompt response normalized directly, so that categories omitted by the production CSV remain available for evaluation. -35. As an evaluator, I want malformed successful responses recorded rather than selectively retried, so that structured-output reliability is part of the observed run. -36. As an evaluator, I want each canonical audit finding to preserve its raw source, prompt, checklist, page, problem, location evidence, WCAG evidence, and parse status, so that comparison is explainable. -37. As a comparison reviewer, I want impossible finding/reference pairs removed deterministically, so that I do not review obviously incompatible matches. -38. As a comparison reviewer, I want plausible candidate pairs shown with evidence from both sides, so that matching decisions are efficient and defensible. -39. As an evaluator, I want a match to require compatible page scope, underlying accessibility failure, and location or element-group evidence, so that superficial wording similarity cannot earn credit. -40. As an evaluator, I want WCAG identifiers and fix recommendations treated as supporting evidence rather than exact-match requirements, so that equivalent findings with different labels can match. -41. As an evaluator, I want one-to-one matching enforced, so that one vague audit finding cannot claim several unrelated workbook rows. -42. As an evaluator, I want multi-element workbook rows treated as one scoring unit when they describe one grouped problem, so that the stakeholder-facing score remains row based. -43. As an evaluator, I want a row containing genuinely different required sub-defects credited only when all required parts match, so that partial detection is not overstated. -44. As an evaluator, I want identical approved normalized inputs to produce exactly the same score, so that scoring is deterministic. -45. As a project lead, I want workbook-row recall shown as caught rows, denominator, and percentage, so that model efficacy is easy to interpret. -46. As a project lead, I want estimated audit cost to be the only ranking tie-breaker, so that cheaper models win only when workbook-row recall is equal. -47. As a project lead, I want latency and token usage recorded but excluded from ranking, so that operational information does not override the agreed efficacy measure. -48. As a project lead, I want unmatched audit findings and structured-output failures reported without changing rank, so that model behavior remains visible without replacing the mandated metric. -49. As a budget owner, I want audit-model cost separated from evaluation-model cost, so that a future Astra reviewer does not distort the model-under-test comparison. -50. As a budget owner, I want total experiment cost reported separately, so that the full research expense remains visible. -51. As a budget owner, I want costs computed from API-reported token categories and a versioned price schedule, so that estimates can be reproduced after prices change. -52. As a budget owner, I want the unrounded estimated audit cost used for ties, so that display rounding cannot change rank. -53. As an operator, I want dry-run behavior by default even when an API key exists, so that environment configuration alone cannot spend money. -54. As an operator, I want live execution to require an explicit flag, so that spending is intentional. -55. As an operator, I want each summarized proof-of-concept run explicitly approved at execution time, so that approval covers a known model, snapshot, request set, configuration, and cost guardrail. -56. As an operator, I want a required maximum audit-cost guardrail, so that a live run stops before starting another request after exhausting its budget. -57. As an operator, I want bounded retries only for transient failures, so that recoverable provider errors do not automatically invalidate a run. -58. As an operator, I want every retry and any billed usage recorded, so that cost and reliability reports are complete. -59. As an evaluator, I want an exhausted transient failure to mark the run incomplete and unranked, so that infrastructure failure is not scored as model ignorance. -60. As an evaluator, I want successful malformed output to produce no usable findings for that prompt, so that model-format failure is represented honestly. -61. As an evaluator, I want end-to-end wall time and per-request duration recorded, so that latency can be analyzed without affecting rank. -62. As an evaluator, I want run manifests to contain content, configuration, code, prompt, parser, and pricing identities, so that only comparable runs are ranked together. -63. As an evaluator, I want incompatible benchmark versions rejected from ordinary comparison, so that changed inputs are not presented as model differences. -64. As a contributor, I want one CLI with prepare, audit, review, and report workflows, so that the evaluation lifecycle is discoverable. -65. As a contributor, I want the evaluation framework to reuse the production audit pipeline through an adapter, so that audit logic is not duplicated. -66. As a contributor, I want the scorer isolated from network and model dependencies, so that it is fast and deterministic to test. -67. As a reviewer, I want reports in JSON, CSV, and Markdown derived from one score object, so that machine, spreadsheet, and human views cannot disagree. -68. As a project lead, I want the homepage result labeled illustrative, so that three provisionally eligible rows are not used to claim broad model equivalence. -69. As a researcher, I want later benchmark expansion organized by page and template family, so that repetitive job pages do not overstate generalization. -70. As a maintainer, I want synthetic fixtures used in committed tests, so that CI never requires private workbook content. -71. As a maintainer, I want ordinary tests and CI to make no live provider calls, so that validation is free, safe, and repeatable. -72. As a maintainer, I want concise errors for missing approvals, incomplete runs, invalid decisions, checksum drift, and match conflicts, so that unsafe states cannot silently produce scores. - -## Implementation Decisions - -- The proof of concept is CLI-only. It will not add controls to the public website or audit API. -- A dedicated model-evaluation module will contain workbook import, benchmark preparation, canonical schemas, audit adaptation, normalization, review, matching, scoring, pricing, and reporting. A single evaluation entry point will expose `prepare`, `audit`, `review`, and `report` subcommands. -- The existing production pipeline remains the audit implementation. The evaluation module will call it through an adapter rather than copying audit logic. -- The external testing seam is the deterministic scorer: approved reference defects, canonical findings, programmatic findings, match decisions, and run metadata enter; one complete score object exits. All reports are projections of that object. -- Two new reviewer seams are required because two adapters are known: human-reviewed local files initially and model-backed review later. `EligibilityReviewer` handles benchmark eligibility decisions; `MatchReviewer` handles per-run match decisions. The scorer does not know which adapter produced an approved decision. -- Human review will use a private Excel workbook with separate Eligibility and Matches sheets. Reviewer adapters emit validated decisions with `accepted`, `rejected`, or `needs_review` state plus identity, rationale, confidence, and timestamp. -- The future Astra adapters will use the same reviewer interfaces and an internal shared OpenAI client. They are designed but disabled for the proof of concept. Workbook-derived data may be sent through the authorized API project, but may not be committed publicly without approval. -- The private source workbook remains at the repository root under its current name by default and is ignored by exact filename. A command-line override permits another local location. The importer validates a recorded SHA-256 and fails clearly on absence or drift. -- All private derived artifacts live under one ignored evaluation workspace partitioned into references, reviews, snapshots, runs, and reports. Only code, schemas, documentation, synthetic fixtures, and public pricing data are committed. -- `openpyxl` is added as a runtime dependency for workbook import and private review workbooks. `pytest` is added as a development dependency. -- The Pristine homepage is captured as raw HTML because that is the production pipeline's input. The capture is immutable and accompanied by source URL, retrieval time, HTTP metadata, and SHA-256. -- The existing DAT Vision Aid HTML fixture is unrelated to the workbook and is never used for workbook-based scoring. -- The proof of concept includes workbook rows scoped as Home and Global on the Pristine homepage. Global rows are labeled as global claims supported by homepage evidence and are not multiplied across pages. -- Every source row becomes one reviewed reference record. Raw text and provenance are retained. Normalized fields include the canonical problem, location or element group, WCAG evidence, eligibility classification, rationale, approval state, and any required sub-defects. -- Eligibility classifications are exactly `llm_eligible`, `programmatic`, `unavailable_evidence`, and `ambiguous`. Only approved `llm_eligible` rows enter workbook-row recall. All classifications remain present in coverage reports. -- The programmatic audit runs first. Its findings are normalized and matched to reference defects through the same evidence-based review process. A programmatic classification does not imply that the checker succeeded; uncovered checker responsibilities are reported as gaps. -- The AI audit preserves current filtering, prompt order, sequential execution, disabled summary prompts, and the 8,192-token output limit. The proof of concept uses GPT-5.6 Luna with medium reasoning, Chat Completions, and no temperature parameter. -- Current audit prompts are unchanged and versioned by content hash. Structured-output enforcement is not enabled in the proof of concept because it would change the audited conditions. -- Raw prompt outputs, request metadata, usage, durations, stop reasons, and failures are saved before normalization. -- Canonical audit findings are built directly from raw prompt responses using fixed per-prompt parsers. The production final CSV, heuristic false-positive suppression, and model-based deduplication are not scoring inputs. -- A canonical audit finding preserves run and model identity, prompt and checklist, page, problem family and statement, element and location evidence, WCAG evidence, raw source finding, and parse status. -- Candidate generation may remove only hard incompatibilities such as different pages or incompatible evidence categories. It does not award matches. -- An approved match requires compatible page or declared scope, the same underlying accessibility failure, and compatible location, element, or element group. WCAG identifiers and remediation wording are supporting evidence. -- Matching is one-to-one at the normalized defect level. Conflicting accepted decisions fail validation rather than being resolved silently. -- One workbook source row remains one headline scoring point. A grouped row is caught when the same grouped defect is identified at the compatible location. A row with distinct required sub-defects is caught only when all required parts are approved as matched. -- The scorer is deterministic and performs no model or network call. Identical normalized inputs and approved decisions produce an identical score object. -- The primary ranking metric is workbook-row recall over approved LLM-eligible rows. Estimated audit cost is the only tie-breaker. No weighted composite score is produced. -- Programmatic coverage, combined workbook coverage, unmatched findings, parse failures, token usage, and latency are reported separately and do not affect model rank. -- Cost reporting separates audit cost, evaluation cost, and total experiment cost. Only audit cost affects ties. Costs use API-reported token categories and a versioned public price schedule; estimates are not represented as invoice totals. -- Latency reporting includes end-to-end audit wall time, per-request durations, and their sum. Latency never affects ranking. -- Evaluation execution is dry by default. An API key in the environment is insufficient to trigger a request. Live execution requires an explicit live flag and a maximum audit-cost limit. -- The proof-of-concept runner permits at most two additional attempts for timeouts, rate limits, and server errors, using bounded exponential backoff. Authentication, invalid-request, parsing, and successful malformed-response failures are not retried. -- Every attempt is recorded. Transient failures exhausted after retries make the run incomplete and unranked. A successful malformed response remains part of the run, contributes no usable findings for that prompt, and is reported as a format failure. -- A run manifest records workbook and snapshot hashes, reference-set version, source-control commit and dirty state, prompt and parser hashes, filters, summaries, model, reasoning effort, endpoint, output limit, retry policy, usage, price-schedule version, timestamps, and run ID. -- Ordinary comparison rejects runs whose snapshot, reference set, prompts, or eligibility set differ. Cross-version analysis must be requested explicitly and labeled accordingly. -- Reports are emitted as JSON, CSV, and Markdown from one score object. The row-level view exposes eligibility, rationale, programmatic outcome, model match, evidence, and final disposition for every in-scope workbook row. -- The proof of concept proceeds through gates: implementation and tests; snapshot and reference preparation; human eligibility approval; evaluation-run planning; point-of-use live-run approval; Luna audit; human match approval; final report. -- One live-run approval covers the complete displayed execution summary, including policy-compliant automatic retries. A materially different configuration or separate rerun requires a newly planned evaluation run and new approval. -- The homepage result is illustrative because the provisional inventory contains only three clearly LLM-eligible rows out of sixteen in-scope rows. Benchmark expansion is required before any small-versus-large model claim. - -## Testing Decisions - -- Tests assert external behavior at module interfaces, not private helper functions or implementation order. The principal scoring seam is the deterministic scorer's structured inputs and returned score object because it exercises eligibility, one-to-one matching, ranking data, coverage, and report inputs at the highest stable seam; the complete operator lifecycle is exercised through the evaluation CLI. -- Workbook-import tests use synthetic Excel fixtures and assert provenance retention, homepage filtering, source-row identity, checksum validation, exact allowed eligibility states, and useful missing-file or drift errors. -- Snapshot tests use local HTTP fakes or saved synthetic responses and assert immutable content, provenance fields, and hashing without accessing the live Pristine site. -- Audit-adapter tests replace the production pipeline with a fake at its existing callable seam. They assert frozen configuration, raw-result preservation, manifest metadata, dry-run safety, live-flag requirements, cost-limit behavior, and retry classification without provider calls. -- Per-prompt normalization is tested through the normalization module's public interface using synthetic raw responses for every active non-summary prompt. Tests include valid arrays and objects, fenced JSON, empty results, malformed successful output, unknown prompt types, and all six categories omitted by the production CSV. -- Reviewer-contract tests run the same decision examples through human-file and fake-model adapters. They assert identical validated outputs, required provenance, allowed states, and rejection of incomplete decisions. -- Candidate and match tests assert that hard incompatibilities are removed, plausible pairs remain reviewable, one-to-one conflicts fail validation, grouped rows receive one point, and distinct required sub-defects require complete coverage. -- Scorer tests assert exact repeatability, correct denominators, binary row credit, separation of programmatic and AI coverage, exclusion of ambiguous and unavailable rows from LLM recall, and cost-only tie-breaking. -- Cost tests use fixed synthetic usage and price schedules. They assert input, cached-input, output, retry, audit, evaluation, and total experiment calculations without depending on current external pricing. -- Failure-policy tests distinguish transient transport failures, exhausted retries, authentication errors, invalid requests, and malformed successful responses. They assert that incomplete runs cannot be ranked. -- Reporting tests construct one score object and assert that JSON, CSV, and Markdown totals and row dispositions agree. -- CLI tests exercise prepare, audit, review, and report workflows against a temporary private workspace with fake adapters. They assert safe defaults, explicit live requirements, and actionable errors. -- Regression fixtures contain no employer workbook text, Pristine content, secrets, or provider responses copied from live runs. -- No ordinary test or CI task may require an API key, access the public internet, or spend money. -- Prior art is the repository's network-free API regression style and CI dry-run smoke path; the evaluation suite extends that philosophy with pytest and synthetic fixtures. - -## Out of Scope - -- Claiming that GPT-5.6 Luna performs as well as a larger model based on the homepage proof of concept. -- Ranking multiple models during the initial no-spend implementation and eligibility-review stage. -- Repeated audit runs, statistical variance analysis, confidence intervals, or stability scoring. -- Using precision, F1, latency, token count, or unmatched-finding count to rank models. -- Automatically accepting Astra eligibility or match decisions during the proof of concept. -- Calling Astra as a reviewer before the human-reviewed process has supplied calibration examples. -- Publishing the source workbook, normalized private reference data, Pristine snapshot, review decisions, raw runs, or reports to Git. -- Redesigning production prompts, filters, payload extractors, programmatic rules, final CSV generation, false-positive suppression, or deduplication. -- Enforcing a new unified structured-output schema on the audit model during the proof of concept. -- Adding evaluation features to the public website or HTTP API. -- Browser rendering, JavaScript execution, screenshots, interactive-state testing, or Chrome-plus-JAWS automation. -- Treating a current live capture as independently proven byte-identical to the September audit state; the benchmark instead records the stakeholder-supported assumption. -- Treating Global rows as verified on every site page. -- Expanding immediately to all seventeen workbook URLs or using repeated job-page rows to claim generalization. -- Computing an exact invoiced dollar amount or accounting for private contractual discounts unavailable from API usage metadata. -- Making a live provider request before eligibility approval, evaluation-run planning, point-of-use approval, and a positive cost limit. - -## Further Notes - -- The provisional homepage inventory contains sixteen workbook rows: five programmatic, three LLM-eligible, six unavailable to the current pipeline, and two ambiguous. These are draft classifications for human review, not accepted benchmark truth. -- Provisional LLM-eligible examples include decorative-image alt judgment and heading-quality problems. Programmatic examples include missing landmarks, missing skip navigation, actionable empty alt, and multiple H1 elements. Unavailable-evidence examples include visual list semantics, styled text that should be headings, carousel controls, Lottie animation, decorative literal glyphs, and media operability. Ambiguities include fragmented adjacent links and a workbook alt-text claim that may conflict with the page. -- The existing committed HTML fixture represents DAT Vision Aid, not Pristine, and cannot be used for this benchmark. -- The workbook is the mandatory provenance source, but approved normalization may mark rows unavailable or ambiguous and may represent distinct required sub-defects. These transformations must remain traceable and reviewable. -- The first implementation milestone ends at the eligibility-review gate and incurs no API cost. The cost limit for the first live Luna run will be selected while planning the evaluation run, whose summary exposes the request set and estimated usage. -- Future benchmark expansion should sample distinct pages and template families rather than random rows because repeated job-page patterns dominate the workbook. - - -## Haiku retarget clarification (2026-09-22) - -The Luna-specific target and credential details above are historical. They are -superseded by `.scratch/anthropic-evaluation-retarget/spec.md`; benchmark, -review, scoring, and safety requirements remain authoritative. diff --git a/.scratch/poc-reduction/spec.md b/.scratch/poc-reduction/spec.md new file mode 100644 index 0000000..f6737c9 --- /dev/null +++ b/.scratch/poc-reduction/spec.md @@ -0,0 +1,118 @@ +# Reduce the accessibility evaluation proof of concept + +Status: ready-for-agent + +## Problem Statement + +The evaluation branch adds 8,764 lines and deletes 635 across 61 files relative to `main` at the time of review. It implements substantially more workflow, compatibility, validation, tests, and documentation than the immediate proof of concept needs. The maintainer needs a small, understandable experiment that works on the given Reference workbook and homepage, rather than a reusable evaluation framework. + +The current generated review workbooks also label the source workbook's `element name` as `location`, implying precision the source does not provide. Eligibility and match review require too much machinery, and following an Evaluation run requires understanding file conventions and validation spread across several modules. + +## Solution + +Reduce the POC to one fixed Haiku evaluation using the given Reference workbook and an existing local HTML Benchmark snapshot. Provide a run, human review, report workflow: preview or explicitly approve paid execution, edit one CSV, then generate one Markdown report. Retain Workbook-row recall, programmatic coverage, Combined workbook coverage, Estimated run cost, and traceable evidence. + +Target fewer than 3,000 added lines relative to `main`, counting code, tests, documentation, configuration, and other tracked changes. Reach that target by deleting unsupported scope and repeated coordination, not compressing code or weakening the meaning of the score. + +## User Stories + +1. As a maintainer, I want fewer than 3,000 added lines against `main`, so that I can review and understand the POC. +2. As an evaluator, I want one complete Haiku evaluation, so that I can demonstrate the measurement method without building model-comparison infrastructure. +3. As an evaluator, I want the importer to support the given Reference workbook directly, so that alternative workbook layouts do not complicate this experiment. +4. As an evaluator, I want every applicable Home and Global row retained, so that the denominator does not silently shrink. +5. As a reviewer, I want original defect text and source sheet and row preserved, so that I can trace each Reference defect to its evidence. +6. As a reviewer, I want the original element name displayed as `source_element`, so that it is not mistaken for an inferred precise location. +7. As a reviewer, I want optional `review_notes`, so that I can clarify location or partial coverage without rewriting source evidence. +8. As an evaluator, I want to supply local HTML and its source URL, so that the evaluator does not need a webpage-capture workflow. +9. As an evaluator, I want the snapshot copied into the Evaluation run with its checksum, so that the evaluated input remains identifiable. +10. As an operator, I want a no-cost preview, so that I can inspect the experiment before spending money. +11. As an operator, I want the fixed model configuration and positive cost guardrail displayed before Live-run approval, so that I understand what I am authorizing. +12. As an operator, I want credentials alone to be insufficient for execution, so that an available key cannot trigger accidental spending. +13. As an operator, I want a fresh directory for each Evaluation run, so that new execution does not overwrite earlier evidence. +14. As an operator, I want execution without a separately saved plan, so that I do not manage plan files or verification stages. +15. As an operator, I want failed requests to stop execution without automatic retries, so that failure behavior and additional spending remain simple. +16. As an evaluator, I want responses and reported usage retained when execution fails, so that unsuccessful work remains inspectable and its reported cost is accounted for. +17. As an evaluator, I want incomplete runs excluded from final scoring, so that partial execution cannot appear to be a complete result. +18. As an operator, I want the cost guardrail checked before each paid request, so that execution stops once accumulated reported cost reaches it. +19. As a reviewer, I want one CSV for eligibility and both finding kinds, so that I do not coordinate multiple review workbooks. +20. As a reviewer, I want to edit that CSV in Excel or a text editor, so that human review does not require a custom interface. +21. As a reviewer, I want classifications and Match decisions made by a human, so that the POC does not require an automated Comparison reviewer. +22. As a reviewer, I want to ask Codex to edit decisions on my behalf when needed, so that assistance uses the same review artifact without adding a reviewer subsystem. +23. As a reviewer, I want each row classified as LLM-eligible, programmatic, unavailable evidence, or ambiguous, so that the report distinguishes responsibility and evidence limitations. +24. As a reviewer, I want short reasons for classifications and matches, so that decisions remain explainable. +25. As a reviewer, I want separate audit and programmatic finding-ID columns, so that either or both finding kinds can cover the same Reference defect. +26. As a reviewer, I want stable finding IDs and inspectable finding evidence, so that I can enter defensible matches without candidate-generation machinery. +27. As a reviewer, I want to associate multiple findings with a row when necessary, so that jointly sufficient evidence is representable. +28. As a reviewer, I want to judge full-row coverage myself, so that the evaluator does not need a required-subdefect model. +29. As a reviewer, I want partial coverage documented in notes without automatically awarding credit, so that Workbook-row recall remains a full-row measure. +30. As an evaluator, I want invoking the report command to confirm completion of human review, so that per-row approval states and reviewer metadata are unnecessary. +31. As an evaluator, I want every row classified before reporting, so that unreviewed eligibility cannot change the denominator unnoticed. +32. As an evaluator, I want blank finding-ID cells to mean not caught when reporting, so that misses have a simple representation. +33. As an evaluator, I want invalid or conflicting finding IDs rejected, so that malformed review data cannot silently inflate coverage. +34. As an evaluator, I want deterministic Workbook-row recall over LLM-eligible rows, so that model efficacy is separate from programmatic coverage. +35. As an evaluator, I want programmatic coverage based on reviewed Programmatic findings, so that classification alone does not award credit. +36. As an evaluator, I want Combined workbook coverage to count each source row at most once, so that overlap between finding kinds does not inflate the result. +37. As an evaluator, I want unavailable and ambiguous rows visible in the report, so that exclusions and evidence limitations remain understandable. +38. As an evaluator, I want Estimated run cost derived from reported usage and recorded pricing, so that the experiment's cost is explainable. +39. As a reader, I want one Markdown report with metrics and row-level explanations, so that I can understand the outcome without reconciling report formats. +40. As an evaluator, I want the review CSV, exact prompts, configuration, raw responses, and pricing retained, so that the report has supporting evidence. +41. As an operator, I want existing private artifacts left untouched, so that simplifying the implementation does not destroy previous work. +42. As a maintainer, I want old-format compatibility removed, so that fresh POC runs do not carry historical workflow complexity. +43. As a maintainer, I want the public application's existing provider behavior preserved, so that reducing the evaluator does not regress unrelated functionality. +44. As a maintainer, I want focused tests and a synthetic run-to-report workflow, so that essential behavior is verified without duplicating the implementation. +45. As a maintainer, I want concise current documentation, so that superseded plans and progress narratives do not dominate the change. + +## Implementation Decisions + +- The supported experiment remains the pinned Haiku configuration already implemented: `claude-haiku-4-5-20251001`, 16,000 thinking tokens, 24,192 total output tokens, and omitted temperature. This reduction does not introduce model selection or change audit prompts, filtering, prompt ordering, or disabled summaries. +- Concentrate Evaluation run orchestration behind the existing CLI seam. Expose run and report operations, with no-cost preview and explicit Live-run approval before billable execution. Choose minimal flag spelling during implementation; obsolete workflow options need no compatibility aliases. +- The run operation imports the known Reference workbook, prepares the existing audit pipeline against the local Benchmark snapshot, performs approved execution, and produces evidence and the review CSV. Human review occurs by editing the CSV; reporting reads that run's evidence and decisions. +- Replace generalized workbook import with explicit mapping of the given sheets and columns. Remove header aliases and elaborate input-recovery messages. Ordinary input failures can fail plainly; retain small checks needed to prevent silently omitted or misidentified reference rows. +- Retain source sheet, row, original defect evidence, and the workbook's identity. Use `source_element` for the original element name and optional `review_notes` for reviewer clarification; do not infer locations automatically. +- Use one CSV row per Reference defect with eligibility, short classification and match reasons, and separate `audit_finding_id` and `programmatic_finding_id` fields. Permit multiple IDs within each finding field using one documented delimiter. Handle CSV quoting and multiline text with standard CSV support. +- Retain the four existing eligibility values: `llm_eligible`, `programmatic`, `unavailable_evidence`, and `ambiguous`. Human classifications are required. Codex assistance is an explicit human-directed edit, not an automated reviewer implementation. +- Remove generated Excel review workbooks, candidate-match generation, reviewer protocols for hypothetical future adapters, per-row approval states, reviewer identity, confidence, timestamps, and required-subdefect machinery. Preserve the substantive ADR-0001 decision: review produces decisions; a separate deterministic scorer consumes them. +- Entered finding IDs assert that the selected findings jointly provide full-row coverage. Partial coverage alone belongs in notes and does not earn credit. Reporting constitutes human confirmation that review is complete; blank finding fields then mean not caught. +- Retain validation of required classifications, row identity and completeness, finding existence and kind, and conflicting reuse of findings across reference rows. Allow multiple findings for one reference and allow both finding kinds to cover the same reference. Keep these checks focused on score correctness rather than general spreadsheet robustness. +- Preserve deterministic scoring semantics: Workbook-row recall measures audit coverage of LLM-eligible rows; programmatic classification alone does not count as coverage; Combined workbook coverage counts the union of covered in-scope rows once. Preserve the existing programmatic-coverage denominator and explicitly report unavailable and ambiguous rows. Remove multi-model ranking machinery without changing ADR-0002's ranking policy for future comparisons. +- Accept a local HTML Benchmark snapshot and source URL; copy the HTML into the fresh run directory and record its checksum. Remove evaluator-owned downloading, HTTP metadata validation, atomic snapshot-publication machinery, and capture recovery paths. +- Save workbook identity and imported reference evidence, copied HTML, exact prompts, actual configuration, raw responses, normalized findings, reported usage, pricing, and completion status. Preserve the existing normalization needed by the actual prompt and programmatic outputs; malformed model output must remain visible rather than silently disappearing. +- Remove separately persisted execution plans, plan reloading, repeated cross-module identity checks, compatibility with old artifact formats, workspace-initialization ceremonies, retries, and resume support. Retain basic run/evidence association needed for correct reporting; recorded provenance need not become a general tamper-verification framework. +- Live execution requires credentials, explicit approval of displayed intent, and a positive cost guardrail. Check accumulated reported cost before every request. This is a between-request guardrail, not a guarantee that an in-flight request cannot exceed the remaining amount. +- Stop on a failed paid request or exhausted cost guardrail; preserve evidence and reported usage obtained so far and mark unfinished runs incomplete. Do not automatically retry failed requests. Incomplete runs cannot produce a final coverage score. +- Keep Haiku usage and pricing handling truthful, including reported usage from interrupted streams and billed output used for thinking. Do not fabricate an unavailable reasoning-token breakdown or persist thinking content as audit evidence. +- Produce one Markdown report containing Workbook-row recall, programmatic coverage, Combined workbook coverage, Estimated run cost, execution/format limitations, and row-level explanations. The editable CSV and finding evidence remain supporting artifacts, not additional report projections. +- Preserve existing private artifacts without migrating, overwriting, or deleting them. The simplified workflow requires fresh runs. Keep private inputs and outputs excluded from version control. +- Preserve shared audit-client behavior required by the public application, including existing providers. Remove evaluator-only compatibility without redesigning or duplicating the shared client. +- Consolidate superseded specifications, progress narratives, and implementation records into concise current guidance where needed to meet the size target; earlier content remains in Git history. Preserve applicable agent instructions and existing ADR decisions. +- Measure added lines in the full branch diff from the merge-base with `main`, including deletions of branch-added material and all retained documentation. Do not game the target through minification, dense statements, hidden generated code, or removal of meaningful tests. If essential behavior prevents the target, report the measured gap and tradeoff. + +## Testing Decisions + +- Prefer the existing CLI seam as the primary test surface: exercise preview, run, edit CSV, and report through observable outputs and saved artifacts. No new public testing interface is required. +- Adapt the existing synthetic CLI lifecycle test to the smaller workflow, using a temporary directory, a synthetic workbook with the actual supported headers, local HTML, and the existing fake-provider seam. Verify exact report metrics and traceable row evidence without private data or paid requests. +- Assert externally visible behavior, not helper calls, internal ordering, private data structures, or exact incidental prose. Remove tests for deleted capabilities rather than keeping a second parallel workflow. +- Test source row inclusion and provenance, `source_element`, multiline CSV round-tripping, required eligibility, misses, multiple finding IDs, valid overlap between finding kinds, and rejected unknown or conflicting IDs. Include a case where an automatically populated blank classification cannot produce a report. +- Verify Workbook-row recall, programmatic coverage, and Combined workbook coverage against hand-calculated examples, preserving existing denominator behavior and handling an empty eligible denominator without inventing a score. Use the existing deterministic scoring tests as prior art; retain focused scorer tests only where they express distinct behavior more clearly than the CLI workflow. +- Verify preview and declined approval make no provider calls; credentials alone do not execute. Verify request failure and budget exhaustion retain available evidence and usage, make no automatic retries, and prevent final scoring of incomplete runs. +- Keep focused existing shared-client tests for thinking-stream final text, interrupted-stream usage, and non-evaluation provider behavior. These test an existing external-provider seam rather than creating another evaluator abstraction. +- Run the repository's applicable checks, verify no private evidence is staged, and report full added-line counts against `main`. Tests and specification publication do not authorize paid evaluation. + +## Out of Scope + +- Supporting arbitrary workbook layouts, extensive Excel validation or recovery, generated Excel review workbooks, or a review application. +- Automated AI eligibility or match reviewers, reviewer-adapter frameworks, and per-row approval/audit-trail metadata. +- Multiple-model comparison, ranking implementation, configurable providers for the evaluator, and changes to existing metric policy. +- Automatic location inference, automated full-row coverage judgment, required-subdefect modeling, and partial-credit scoring. +- Webpage capture, browser automation, HTTP provenance frameworks, persisted execution plans, automatic retries, and resumable execution. +- Migration or reporting compatibility for historical evaluation formats; deletion of existing private artifacts. +- Separate JSON or CSV report projections, unrelated public application changes, and changing the audit prompts to fit the evaluator. +- Performing a paid run as part of implementing or verifying this reduction. + +## Further Notes + +This specification supersedes earlier evaluation requirements only where they conflict with the agreed POC reduction. It retains review/scoring separation and Workbook-row recall policy from the existing ADRs; removing future reviewer machinery and ranking code does not replace those policies. + +Inspection found 16 imported Reference defects in the current private artifacts. Their generated eligibility-workbook locations and approved-reference locations matched the imported values, which came from the source `element name` field. This motivates preserving the source label rather than generating a new location. + +The agreed size target is fewer than 3,000 added lines for the whole branch change, not merely the evaluation implementation. The existing synthetic CLI workflow and fake-provider seam are the proposed highest practical test seam. Publication marks the work ready for implementation; this task changes the specification only and does not claim the reduction is implemented or verified. diff --git a/CONTEXT.md b/CONTEXT.md index 1dbdc14..ca2be15 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,77 +1,55 @@ # Accessibility Audit Evaluation -This context describes the language used to evaluate AI-generated accessibility audit results against a human-reviewed reference. +This context names the evidence and measures for the homepage accessibility experiment. ## Language **Reference workbook**: -The authorized Excel workbook containing the human accessibility audit. Its worksheets provide source evidence for reference defects; generated eligibility and match workbooks are review artifacts, not the reference workbook. -_Avoid_: Ground-truth spreadsheet, review workbook, evaluation report +The authorized human accessibility audit workbook that supplies source evidence. +_Avoid_: Ground-truth spreadsheet, generated review workbook. **Reference defect**: -A human-reviewed accessibility problem derived from the mandated defect-report workbook and retained as benchmark evidence with traceable source provenance. -_Avoid_: Ground-truth row, Excel issue, expected finding - -**Audit finding**: -An accessibility problem reported by the AI audit model under evaluation. Programmatic checker output is not an audit finding for the primary model-efficacy score. -_Avoid_: Reference defect, programmatic finding, model response - -**Workbook-row recall**: -The count and proportion of scorable workbook source rows caught by correctly matched audit findings. It is the primary model-ranking metric; cost is the only tie-breaker. -_Avoid_: Precision, F1, aggregate quality score - -**LLM-eligible reference defect**: -A reference defect for which the AI audit model receives enough evidence and responsibility to make the relevant accessibility judgment. Defects intentionally handled only by programmatic checks are not LLM-eligible. -_Avoid_: Every workbook row, programmatic defect - -**Programmatic reference defect**: -A reference defect assigned to deterministic audit checks rather than the AI audit model. Classification alone does not award coverage; an approved programmatic match is still required. -_Avoid_: Programmatic finding, automatically covered defect - -**Unavailable-evidence reference defect**: -A reference defect that the current audit pipeline cannot judge because its inputs omit required visual, styling, interaction, cross-page, or other evidence. -_Avoid_: Model miss, invalid reference defect - -**Ambiguous reference defect**: -A reference defect whose claim, scope, or required sub-defects cannot yet be interpreted confidently enough for fair responsibility assignment or scoring. An accepted ambiguous classification is a completed review decision, not an unresolved review state. -_Avoid_: Model uncertainty, unavailable-evidence reference defect +One source workbook defect retained with its original evidence and sheet and row provenance. Its `source_element` is the original element name, not a precise inferred location. +_Avoid_: Expected finding, inferred location. **Benchmark snapshot**: -An immutable capture of the audited webpage used as the common input for every evaluated model, with its source and capture identity recorded. -_Avoid_: Live page, test URL +The identified HTML evidence for the page being audited, paired with its source URL and checksum. +_Avoid_: Live page. **Evaluation run**: -One planned model evaluation and its resulting audit evidence, reviewed match decisions, and reports, identified together for reproducibility. -_Avoid_: Dry-run folder, report bundle +One fixed-model audit and its evidence, human decisions, completion status, and report. +_Avoid_: Execution plan, report bundle. -**Eligibility classification**: -The reviewed assignment explaining whether a reference defect belongs to the AI model's responsibility, a programmatic check, unavailable audit evidence, or unresolved ambiguity. -_Avoid_: Model score, exclusion without rationale +**Audit finding**: +An accessibility problem reported by the AI audit model and retained with its source response. +_Avoid_: Reference defect, programmatic finding. **Programmatic finding**: -An accessibility problem reported by a deterministic checker rather than an AI audit model. It contributes to combined workbook coverage but not to model ranking. -_Avoid_: Audit finding, LLM-eligible reference defect +An accessibility problem reported by a deterministic checker rather than the AI audit model. +_Avoid_: Audit finding. -**Combined workbook coverage**: -The count and proportion of in-scope workbook rows caught by either programmatic findings or audit findings, with unavailable and ambiguous rows still reported explicitly. -_Avoid_: Workbook-row recall, model score - -**Canonical audit finding**: -A lossless, structured representation of one raw audit finding used for comparison across prompts and models. -_Avoid_: Production report row, raw model response +**Eligibility classification**: +The human assignment of a Reference defect to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with a reason. +_Avoid_: Automatic model score, exclusion without rationale. **Match decision**: -An approved determination that one canonical audit finding does or does not correspond to one reference defect, with supporting evidence and rationale. -_Avoid_: Candidate match, similarity score +A human judgment that selected findings jointly provide full-row coverage of one Reference defect. Partial evidence alone does not award credit. +_Avoid_: Candidate match, partial-credit score. + +**Workbook-row recall**: +The proportion of LLM-eligible Reference defects fully covered by Audit findings. +_Avoid_: Precision, F1, aggregate quality score. + +**Programmatic coverage**: +The proportion of programmatic Reference defects fully covered by Programmatic findings. Classification alone does not award coverage. + +**Combined workbook coverage**: +The proportion of imported Reference defects fully covered by either finding kind, counting each row once. +_Avoid_: Workbook-row recall, model score. **Estimated run cost**: -The calculated cost of one audit run using API-reported token usage and a recorded model-price schedule. It is the sole tie-breaker when workbook-row recall is equal. -_Avoid_: Invoice amount, latency score +The cost calculated from reported usage and a recorded pricing schedule, rather than an invoice amount. **Live-run approval**: -An operator's explicit confirmation to execute a displayed audit plan under a maximum audit-cost guardrail. -_Avoid_: Authorization digest, API-key presence - -**Comparison reviewer**: -A human or model that proposes eligibility classifications or match decisions through the same review process. A comparison reviewer does not calculate the final score. -_Avoid_: Scorer, audit model +An operator's explicit authorization of a displayed fixed-model execution under a positive between-request cost guardrail. +_Avoid_: API-key presence. diff --git a/README.md b/README.md index 931e284..775dba0 100644 --- a/README.md +++ b/README.md @@ -239,152 +239,59 @@ GEMINI_API_KEY=AIza... ## Private Model Evaluation -The evaluation CLI keeps the employer-provided workbook and every derived -artifact outside version control. Obtain -`Pristine Accessibility Defect Report.xlsx` from an authorized project source, -place it in the repository root, and initialize the private workspace: +The private POC measures one fixed Haiku audit against the Home and Global +rows of the authorized Reference workbook. Obtain the workbook from the project +source; the default filename is `Pristine Accessibility Defect Report.xlsx`. +All run evidence stays under ignored `.model-evaluation/runs/`. -```bash -uv run visionaid-evaluate prepare -``` - -To freeze the Pristine homepage for model comparisons, supply its authorized -URL during preparation: - -```bash -uv run visionaid-evaluate prepare \ - --snapshot-url "$PRISTINE_HOMEPAGE_URL" -``` - -The command saves the HTTP response body without HTML transformation as -`.model-evaluation/snapshots/pristine-homepage/source.html`. The adjacent -`metadata.json` records the source and final URLs, retrieval time, HTTP status, -request user agent, response identity headers, byte length, and SHA-256. Both -files are staged and published as one immutable bundle. Repeating preparation -for the same URL verifies and reuses that bundle without another request. -Checksum drift, changed provenance, or a partial snapshot fails closed; prepare -never silently replaces an existing benchmark. Local HTML files, including the -unrelated DAT Vision Aid fixture, cannot be used as the Pristine snapshot. - -Use `--workbook PATH` for another authorized local copy outside the repository. -The fixed `.model-evaluation/` workspace contains separate `references/`, -`reviews/`, `snapshots/`, `runs/`, and `reports/` partitions, all covered by one -ignore rule. - -Run `uv run visionaid-evaluate --help` to discover the `prepare`, `audit`, -`review`, and `report` workflows. To import the homepage rows, record the -authorized workbook checksum and canonical homepage URL. This creates a -lossless private reference set and an Eligibility workbook: - -```bash -uv run visionaid-evaluate prepare \ - --workbook-sha256 "$AUTHORIZED_WORKBOOK_SHA256" \ - --homepage-url "$PRISTINE_HOMEPAGE_URL" -``` - -Complete every row in -`.model-evaluation/reviews/eligibility.xlsx`, including classification, -decision, rationale, reviewer, confidence, and timestamp, then validate it: - -```bash -uv run visionaid-evaluate review \ - --eligibility-workbook .model-evaluation/reviews/eligibility.xlsx -``` - -Only the four classifications `llm_eligible`, `programmatic`, -`unavailable_evidence`, and `ambiguous` are accepted. Global rows remain one -workbook-row scoring unit supported by homepage evidence. - -`audit` is network-free by default even if `ANTHROPIC_API_KEY` exists. With an -approved reference set and benchmark snapshot it runs the production -extractors, filters, programmatic checks, slicers, and unchanged prompts to -create a uniquely identified evaluation run. Planning requires a positive -maximum audit cost and freezes that guardrail and the approved reference set -inside the run directory: - -```bash -uv run visionaid-evaluate audit \ - --max-audit-cost-usd "$PROPOSED_LIMIT" -``` - -Keep the printed `.model-evaluation/runs/` path. The summary shows the -model, endpoint, thinking budget and total output cap, benchmark and reference identities, prompt -request set, estimated input usage, retry policy, saved cost guardrail, and -destination. A billable run requires `--live`, that run directory, and -`ANTHROPIC_API_KEY`. The command revalidates the frozen evidence, prints the same -summary, and asks a terminal operator to type `yes`: +Preview the exact prompt set and estimated paid intent with a local HTML +Benchmark snapshot and the URL it represents: ```bash -uv run visionaid-evaluate audit \ - --live \ - --run-dir .model-evaluation/runs/ +uv run visionaid-evaluate run --html /path/to/home.html \ + --source-url https://pristineai.com/ --max-cost-usd 1.00 ``` -Deliberate automation may add `--auto-approve`. It skips terminal input but -does not skip the summary, API credential, integrity checks, saved cost limit, -or overwrite protection. Any rerun requires planning a fresh evaluation run. - -The runner uses pinned `claude-haiku-4-5-20251001` through Anthropic Messages -with manual extended thinking capped at 16,000 tokens and a total output cap -of 24,192 tokens (up to 8,192 for final text after the full thinking budget). -It omits temperature, keeps summaries disabled, and streams internally. Only -final audit text is retained; thinking, signatures, and redacted thinking are -discarded. Aggregate output usage includes billed thinking; a separate reasoning -token count is unavailable. `pricing.v2.json` records the official direct API -rates ($1/M input, $5/M output), source, and verification date. The plan and -approval summary display the configuration, pricing identity, and cost guardrail. -Historical Luna runs remain readable, but cannot be executed under this target. -It preserves every attempt and final response. It retries only transient timeout, rate-limit, and server -failures (at most twice). Exhausted failures make a run incomplete and -unrankable; successful malformed JSON is retained as a format failure and is -not retried. - -Canonical audit findings and programmatic findings use separate, run-scoped -Matches workbooks. Generate, complete, and import both review kinds: +The CLI copies the HTML, records its SHA-256 and workbook identity, imports +all Home and Global defect rows, runs the existing programmatic checks, and +saves the exact filtered prompts. Preview makes no provider calls even when +credentials are available. It creates a fresh run directory and prints its +path. Use `--workbook PATH` to select another authorized copy of the same +workbook layout. + +For a paid run, repeat the command with `--live`. The CLI shows the fixed +`claude-haiku-4-5-20251001` configuration (16,000 thinking tokens, 24,192 +total output tokens, no temperature), prompt count, pricing, and positive +cost guardrail before asking you to type `yes`. `--approve-live` is an +explicit approval flag for deliberate noninteractive execution. An API key +alone never starts paid requests. The guardrail is checked before each +request; a single request can exceed the remaining amount. Failed requests +are not retried. Responses and reported usage are saved as they arrive, and +incomplete runs cannot be scored. Thinking content is not retained. + +Edit the printed `review.csv` in Excel or a text editor. Classify **every** +row as `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, +and give a short `classification_reason`. Enter full-row matches in +`audit_finding_id` and/or `programmatic_finding_id`; separate multiple IDs +with semicolons and provide a `match_reason`. Leave both ID cells blank for a +miss. Use `review_notes` for partial coverage or location clarification. +`source_element` is the workbook's original `element name`, not an inferred +precise location. Finding IDs and raw evidence are in `audit-findings.json` +and `programmatic-findings.json`. Codex can edit the same CSV on request. + +After human review, generate the Markdown report: ```bash -uv run visionaid-evaluate review \ - --run-dir .model-evaluation/runs/ \ - --kind audit -uv run visionaid-evaluate review \ - --run-dir .model-evaluation/runs/ \ - --kind audit \ - --import-matches -uv run visionaid-evaluate review \ - --run-dir .model-evaluation/runs/ \ - --kind programmatic -uv run visionaid-evaluate review \ - --run-dir .model-evaluation/runs/ \ - --kind programmatic \ - --import-matches +uv run visionaid-evaluate report --run-dir .model-evaluation/runs/ ``` -The conventional workbooks and decision files live under -`.model-evaluation/reviews//`. Explicit `--reference-set`, `--findings`, -`--matches-workbook`, and `--match-decisions` overrides remain available for -investigation and recovery. Accepted matches require compatible page scope, -underlying failure, and location or element-group evidence; WCAG labels and -remediation wording are supporting evidence. - -After both reviews, reporting discovers the frozen references, both finding -collections, both decision files, the live manifest, and format failures from -the evaluation run: - -```bash -uv run visionaid-evaluate report \ - --run-dir .model-evaluation/runs/ -``` - -The scorer is network-free, enforces one-to-one matching and all required -sub-defects, and emits JSON, CSV, and Markdown from one score object. Missing -review artifacts produce the exact next review command instead of requiring a -hand-written report bundle. - -Workbook-row recall over approved LLM-eligible rows is the primary ranking -metric. Unrounded estimated audit cost is the only tie-breaker. Programmatic -and combined coverage, tokens, latency, evaluation cost, unmatched findings, -and format failures remain informational. The homepage result is illustrative -and cannot establish broad small-versus-large model equivalence. +The report gives Workbook-row recall for LLM-eligible rows, programmatic +coverage for programmatic rows, Combined workbook coverage over all imported +rows, Estimated run cost from API-reported usage and saved pricing, and +source-row explanations. Blank IDs count as misses. A zero eligible +denominator is shown as N/A. This one-homepage result is illustrative and +does not establish broad model equivalence. Historical private runs remain +untouched; the reduced CLI requires fresh runs. ## Running the Web App diff --git a/docs/adr/0001-separate-review-from-scoring.md b/docs/adr/0001-separate-review-from-scoring.md index cdc2e48..dbc53a3 100644 --- a/docs/adr/0001-separate-review-from-scoring.md +++ b/docs/adr/0001-separate-review-from-scoring.md @@ -1,3 +1,3 @@ # Separate review from deterministic scoring -The evaluation framework separates eligibility and match review from scoring: `EligibilityReviewer` and `MatchReviewer` produce structured decisions through human-reviewed file adapters initially and optional model-backed adapters later, while the scorer consumes only approved decisions and remains deterministic. This keeps identical normalized inputs exactly reproducible, lets an Astra reviewer replace tedious human review without rewriting import, scoring, or reporting, and allows workbook-derived review artifacts to remain private even though authorized models may process them. +A human records eligibility classifications and full-row Match decisions in the review CSV; a separate deterministic scorer consumes those decisions. This keeps the meaning of coverage under human control while making the same reviewed evidence reproducible without an automated reviewer subsystem. diff --git a/pyproject.toml b/pyproject.toml index c3e4ce7..6de5cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.v1.json", "pricing.v2.json"] +"vision_aid.evaluation" = ["pricing.v2.json"] "processing_scripts.llm" = ["*.txt"] diff --git a/tests/test_evaluation_audit.py b/tests/test_evaluation_audit.py deleted file mode 100644 index 3eb463b..0000000 --- a/tests/test_evaluation_audit.py +++ /dev/null @@ -1,594 +0,0 @@ -import hashlib -import json -import shutil -from pathlib import Path - -import pytest - -from vision_aid.evaluation.audit import ( - AuditExecutionError, - build_audit_plan, - execute_audit_run, - load_audit_plan, - load_verified_audit_plan, -) -from vision_aid.evaluation.pricing import PriceSchedule -from vision_aid.evaluation.schemas import ReferenceSet -from vision_aid.evaluation.serialization import ( - reference_set_identity, - save_reference_set, -) - -REFERENCE_SET = ReferenceSet( - version="references-v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=(), -) -REFERENCE_IDENTITY = reference_set_identity(REFERENCE_SET) -APPROVED_AT = "2026-09-19T12:00:00Z" - - -def _reference_set_path(directory: Path) -> Path: - """Write the frozen synthetic reference set used by an audit plan.""" - path = directory / "reference-set.json" - save_reference_set(path, REFERENCE_SET) - return path - - -def _fake_pipeline( - html_path: str, - output_dir: Path, - api_key: str | None, - model: str, - dry_run: bool, - include_summaries: bool, - progress_callback=None, - request_config=None, -) -> dict: - """Write two deterministic prompt artifacts through the adapter seam.""" - assert api_key is None - assert dry_run is True - assert include_summaries is False - assert model == "claude-haiku-4-5-20251001" - assert request_config.thinking_budget_tokens == 16000 - assert request_config.temperature is None - assert request_config.max_output_tokens == 24192 - prompts = output_dir / "prompts" - prompts.mkdir(parents=True) - entries = [] - for index, name in enumerate(("heading_structure", "link_clarity"), start=1): - prompt_text = f"prompt {index}" - (prompts / f"{name}.json").write_text( - json.dumps( - { - "prompt_name": name, - "checklist": "CL01", - "wcag_criteria": ["1.3.1"], - "prompt_index": index, - "prompt_tokens_est": 10 * index, - "prompt_text": prompt_text, - "payload_slice": "{}", - } - ), - encoding="utf-8", - ) - entries.append( - { - "name": name, - "checklist": "CL01", - "wcag_criteria": ["1.3.1"], - "prompt_tokens_est": 10 * index, - "status": "dry_run", - } - ) - (output_dir / "programmatic_findings.json").write_text("[]", encoding="utf-8") - return { - "html_file": html_path, - "model": model, - "dry_run": True, - "include_summaries": False, - "prompts_dry_run": entries, - "prompts_executed": [], - "prompts_skipped": [], - "pass1_filters_active": [], - } - - -def test_dry_plan_freezes_configuration_content_and_code_identities( - tmp_path: Path, -) -> None: - """A dry plan records the complete frozen benchmark configuration.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("Synthetic", encoding="utf-8") - snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() - run_directory = tmp_path / "run" - - plan = build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1.00", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=snapshot_sha256, - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - - assert plan["configuration"] == { - "endpoint": "messages", - "include_summaries": False, - "max_output_tokens": 24192, - "model": "claude-haiku-4-5-20251001", - "provider": "anthropic", - "thinking": {"type": "enabled", "budget_tokens": 16000}, - "temperature": None, - "execution": "sequential", - } - assert plan["request_count"] == 2 - assert plan["estimated_input_tokens"] == 30 - assert plan["maximum_audit_cost_usd"] == "1" - assert list(plan["prompt_hashes"]) == ["heading_structure", "link_clarity"] - assert len(plan["parser_hash"]) == 64 - assert len(plan["pricing_identity"]) == 64 - assert len(plan["plan_digest"]) == 64 - assert (run_directory / "evaluation-manifest.json").is_file() - - -@pytest.mark.parametrize("limit", ("0", "-1", "NaN", "not-a-number")) -def test_audit_plan_requires_a_positive_finite_cost_limit( - tmp_path: Path, - limit: str, -) -> None: - """Planning rejects absent economic meaning before running the pipeline.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "run" - - with pytest.raises(AuditExecutionError, match="cost limit"): - build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd=limit, - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - - -class FakeClient: - def __init__(self, outcomes: list[dict]): - """Queue provider outcomes in request order.""" - self.outcomes = outcomes - - def call(self, prompt: str) -> dict: - """Return the next provider outcome without network access.""" - return self.outcomes.pop(0) - - -def _success(response: str, input_tokens: int = 10, output_tokens: int = 5) -> dict: - """Return one successful synthetic provider result.""" - return { - "success": True, - "response": response, - "model": "claude-haiku-4-5-20251001", - "provider": {"name": "anthropic", "endpoint": "messages"}, - "usage": { - "input_tokens": input_tokens, - "cached_input_tokens": 0, - "output_tokens": output_tokens, - "reasoning_tokens": None, - "cache_creation_input_tokens": 0, - "total_tokens": input_tokens + output_tokens, - "provider_usage": {}, - }, - "stop_reason": "stop", - "duration_seconds": 0.2, - "failure": None, - } - - -def _transient() -> dict: - """Return one billed, retryable rate-limit result.""" - return { - "success": False, - "response": None, - "model": "claude-haiku-4-5-20251001", - "provider": {"name": "anthropic", "endpoint": "messages"}, - "usage": { - "input_tokens": 2, - "cached_input_tokens": 0, - "output_tokens": 0, - "reasoning_tokens": 0, - "cache_creation_input_tokens": 0, - "total_tokens": 2, - "provider_usage": {}, - }, - "stop_reason": None, - "duration_seconds": 0.1, - "failure": {"type": "RateLimitError", "status_code": 429}, - "error": "rate limited", - } - - -def _permanent(status_code: int, failure_type: str) -> dict: - """Return an unbilled provider rejection that must not be retried.""" - result = _transient() - result["usage"]["input_tokens"] = 0 - result["usage"]["total_tokens"] = 0 - result["failure"] = { - "type": failure_type, - "status_code": status_code, - } - return result - - -def test_live_gates_retries_and_format_failures( - tmp_path: Path, -) -> None: - """A live run enforces gates and preserves retries and malformed output.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() - run_directory = tmp_path / "run" - plan = build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=snapshot_sha256, - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - with pytest.raises(AuditExecutionError, match="--live"): - execute_audit_run( - run_directory, - live=False, - api_key="secret", - approval_mode="interactive", - approved_at=APPROVED_AT, - client=FakeClient([]), - sleep=lambda _delay: None, - ) - - with pytest.raises(AuditExecutionError, match="approval"): - execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="wrong", - approved_at=APPROVED_AT, - client=FakeClient([]), - sleep=lambda _delay: None, - ) - - client = FakeClient( - [ - _transient(), - _success('[{"problem":"Heading skips a level"}]'), - _success("not json"), - ] - ) - manifest = execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="interactive", - approved_at=APPROVED_AT, - client=client, - sleep=lambda _delay: None, - ) - - assert manifest["complete"] is True - assert manifest["rankable"] is True - assert len(manifest["requests"][0]["attempts"]) == 2 - assert manifest["format_failures"] == ["link_clarity"] - assert manifest["usage"]["input_tokens"] == 22 - assert manifest["usage"]["reasoning_tokens"] is None - assert manifest["estimated_audit_cost_usd"] == "0.000072" - assert manifest["configuration"]["temperature"] is None - assert manifest["approval"] == { - "mode": "interactive", - "approved_at": APPROVED_AT, - } - assert "authorization_digest" not in manifest - raw_attempts = json.loads( - (run_directory / "raw-attempts.json").read_text(encoding="utf-8") - ) - assert len(raw_attempts["requests"][0]["attempts"]) == 2 - - with pytest.raises(AuditExecutionError, match="fresh evaluation run"): - execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="interactive", - approved_at=APPROVED_AT, - client=FakeClient([]), - sleep=lambda _delay: None, - ) - - -@pytest.mark.parametrize( - "changed_artifact", - ("plan", "snapshot", "reference", "prompt"), -) -def test_live_execution_revalidates_frozen_evidence_before_requests( - tmp_path: Path, - changed_artifact: str, -) -> None: - """Any planned evidence drift fails closed before a provider request.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "run" - reference_path = _reference_set_path(run_directory) - build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=reference_path, - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - if changed_artifact == "plan": - path = run_directory / "evaluation-manifest.json" - value = json.loads(path.read_text(encoding="utf-8")) - value["maximum_audit_cost_usd"] = "2" - path.write_text(json.dumps(value), encoding="utf-8") - elif changed_artifact == "snapshot": - snapshot.write_text("changed", encoding="utf-8") - elif changed_artifact == "reference": - reference_path.write_text("{}", encoding="utf-8") - else: - prompt_path = run_directory / "prompts" / "heading_structure.json" - value = json.loads(prompt_path.read_text(encoding="utf-8")) - value["prompt_text"] = "changed" - prompt_path.write_text(json.dumps(value), encoding="utf-8") - client = FakeClient([]) - - with pytest.raises(AuditExecutionError): - execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="auto", - approved_at=APPROVED_AT, - client=client, - sleep=lambda _delay: None, - ) - - assert client.outcomes == [] - - -def test_copied_run_cannot_be_executed_as_a_fresh_destination( - tmp_path: Path, -) -> None: - """Copying an unexecuted run cannot create another billable run.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "workspace" / "runs" / "run" - build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - copied_run = tmp_path / "other-workspace" / "runs" / "run" - shutil.copytree(run_directory, copied_run) - client = FakeClient([]) - - with pytest.raises(AuditExecutionError, match="live destination"): - execute_audit_run( - copied_run, - live=True, - api_key="secret", - approval_mode="auto", - approved_at=APPROVED_AT, - client=client, - sleep=lambda _delay: None, - ) - - assert client.outcomes == [] - - -def test_historical_plan_loading_does_not_require_current_execution_inputs( - tmp_path: Path, -) -> None: - """Review and reporting can use a run after pricing or snapshot changes.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "run" - current_schedule = PriceSchedule.default() - historical_schedule = PriceSchedule( - data=current_schedule.data, - identity="historical-pricing-identity", - ) - build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - schedule=historical_schedule, - ) - snapshot.unlink() - - assert load_audit_plan(run_directory)["run_id"] == "run" - with pytest.raises(AuditExecutionError): - load_verified_audit_plan(run_directory) - - -def test_cost_guard_is_checked_before_each_retry(tmp_path: Path) -> None: - """A billed failure can exhaust the limit before an automatic retry.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - snapshot_sha256 = hashlib.sha256(snapshot.read_bytes()).hexdigest() - run_directory = tmp_path / "run" - limit = "0.0000001" - plan = build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd=limit, - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=snapshot_sha256, - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - client = FakeClient([_transient(), _success("[]")]) - - manifest = execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="auto", - approved_at=APPROVED_AT, - client=client, - sleep=lambda _delay: None, - ) - - assert manifest["complete"] is False - assert manifest["incomplete_reason"] == "audit_cost_limit_exhausted" - assert len(manifest["requests"]) == 1 - assert len(manifest["requests"][0]["attempts"]) == 1 - assert len(client.outcomes) == 1 - - -@pytest.mark.parametrize( - "outcome,expected", - [ - (_permanent(401, "AuthenticationError"), "authentication"), - (_permanent(400, "BadRequestError"), "invalid_request"), - ], -) -def test_permanent_provider_failures_are_not_retried( - tmp_path: Path, - outcome: dict, - expected: str, -) -> None: - """Authentication and invalid requests stop after one recorded attempt.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "run" - plan = build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - client = FakeClient([outcome, _success("[]")]) - - manifest = execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="auto", - approved_at=APPROVED_AT, - client=client, - sleep=lambda _delay: None, - ) - - assert manifest["incomplete_reason"] == expected - assert len(manifest["requests"][0]["attempts"]) == 1 - assert len(client.outcomes) == 1 - - -def test_exhausted_transient_failures_make_the_run_unranked( - tmp_path: Path, -) -> None: - """Record three transient attempts before marking the run incomplete.""" - snapshot = tmp_path / "source.html" - snapshot.write_text("", encoding="utf-8") - run_directory = tmp_path / "run" - plan = build_audit_plan( - snapshot, - run_directory, - maximum_audit_cost_usd="1", - reference_set_path=_reference_set_path(run_directory), - workbook_sha256="a" * 64, - snapshot_sha256=hashlib.sha256(snapshot.read_bytes()).hexdigest(), - reference_set_version="references-v1", - eligibility_identity=REFERENCE_IDENTITY, - pipeline=_fake_pipeline, - repository=tmp_path, - ) - - manifest = execute_audit_run( - run_directory, - live=True, - api_key="secret", - approval_mode="auto", - approved_at=APPROVED_AT, - client=FakeClient([_transient(), _transient(), _transient()]), - sleep=lambda _delay: None, - ) - - assert manifest["complete"] is False - assert manifest["rankable"] is False - assert manifest["incomplete_reason"] == "transient_failures_exhausted" - assert len(manifest["requests"][0]["attempts"]) == 3 - - -def test_versioned_price_schedule_uses_reported_token_categories() -> None: - """Pricing distinguishes uncached, cached, written, and output tokens.""" - schedule = PriceSchedule.default() - assert ( - schedule.estimate( - "gpt-5.6-luna", - { - "input_tokens": 100, - "cached_input_tokens": 40, - "cache_creation_input_tokens": 20, - "output_tokens": 10, - "reasoning_tokens": 5, - }, - ) - == "0.0000298" - ) - - -def test_haiku_prices_all_output_including_thinking(): - assert ( - PriceSchedule.default().estimate( - "claude-haiku-4-5-20251001", - { - "input_tokens": 100, - "output_tokens": 16002, - "reasoning_tokens": None, - }, - ) - == "0.08011" - ) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 706a745..22b331d 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,1008 +1,233 @@ +"""Run, review, and report through the CLI and its saved artifacts.""" + +import csv import hashlib -import io import json -import os -import subprocess -import sys -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from datetime import datetime -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from tempfile import TemporaryDirectory -from threading import Thread import openpyxl import pytest -from vision_aid.evaluation import audit as evaluation_audit -from vision_aid.evaluation import cli as evaluation_cli -from vision_aid.evaluation.cli import main as evaluation_main -from vision_aid.evaluation.schemas import ReferenceSet -from vision_aid.evaluation.serialization import save_reference_set -from vision_aid.evaluation.workspace import PrivateWorkspace - -PROJECT_ROOT = Path(__file__).parents[1] -SNAPSHOT_BODY = ( - b'\r\nPristine' - b"caf\xc3\xa9\r\n" -) -SNAPSHOT_SHA256 = "aadf66066b05d9c8d8268ba7e28ebcef8a2ed81f672fc2e2285d9236be00e89d" - - -class SnapshotHTTPServer(ThreadingHTTPServer): - """Serve one synthetic HTML response and record benchmark fetches.""" - - request_count = 0 - request_user_agents: list[str | None] - on_request: Callable[[], None] | None - - -class SnapshotHandler(BaseHTTPRequestHandler): - """Return a byte-sensitive local response without external network access.""" - - def do_GET(self) -> None: - """Serve the synthetic homepage and record acquisition metadata.""" - server = self.server - assert isinstance(server, SnapshotHTTPServer) - server.request_count += 1 - server.request_user_agents.append(self.headers.get("User-Agent")) - if server.on_request is not None: - server.on_request() - body = SNAPSHOT_BODY - content_type = "text/html; charset=utf-8" - if self.path == "/non-utf8": - body = b"caf\xe9" - content_type = "text/html; charset=iso-8859-1" - self.send_response(200) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(body))) - self.send_header("ETag", '"synthetic-v1"') - self.send_header("Last-Modified", "Wed, 16 Sep 2026 14:00:00 GMT") - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - """Keep the test output free of local server access logs.""" - - -@contextmanager -def local_snapshot_server( - on_request: Callable[[], None] | None = None, -) -> Iterator[tuple[str, SnapshotHTTPServer]]: - """Run the synthetic benchmark source on an ephemeral loopback port.""" - server = SnapshotHTTPServer(("127.0.0.1", 0), SnapshotHandler) - server.request_user_agents = [] - server.on_request = on_request - thread = Thread(target=server.serve_forever) - thread.start() - host, port = server.server_address - try: - yield f"http://{host}:{port}/pristine-homepage", server - finally: - server.shutdown() - thread.join() - server.server_close() - - -def run_cli( - *arguments: str, - cwd: Path = PROJECT_ROOT, - env: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - """Run the installed evaluation module through its public CLI boundary.""" - command_env = os.environ.copy() - if env: - command_env.update(env) - return subprocess.run( - [sys.executable, "-m", "vision_aid.evaluation.cli", *arguments], - cwd=cwd, - env=command_env, - capture_output=True, - text=True, - check=False, - ) - - -class TerminalInput(io.StringIO): - """Provide controlled input with an explicit terminal capability.""" - - def __init__(self, value: str, *, terminal: bool = True) -> None: - super().__init__(value) - self.terminal = terminal - - def isatty(self) -> bool: - """Report whether interactive confirmation is available.""" - return self.terminal - - -def test_audit_loads_the_project_env_explicitly( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The evaluation audit keeps local .env support without import effects.""" - loaded_paths: list[Path] = [] - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(evaluation_cli, "_PROJECT_ENV_LOADED", False) - monkeypatch.setattr( - evaluation_cli, - "load_dotenv", - lambda path: loaded_paths.append(Path(path)), - ) - - with pytest.raises(SystemExit) as exit_info: - evaluation_main(["audit"]) - with pytest.raises(SystemExit): - evaluation_main(["audit"]) +from vision_aid.evaluation import audit +from vision_aid.evaluation.cli import main - assert exit_info.value.code == 2 - assert loaded_paths == [PROJECT_ROOT / ".env"] +def inputs(tmp_path: Path) -> tuple[Path, Path]: + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "Defect report" + sheet.append(["Sr. #", "element name", "Browser Combination", "Page name", + "Issue Title", "Steps to Reproduce", "actual result", + "expected result", "Recommendation for Fix", "WCAG Sc", + "Type of Change", "comment"]) + sheet.append([1, "First link", "Chrome", "Home", "Unclear link", "Open page", + "Vague link\nfor everyone", "Descriptive text", "Rename", "2.4.4", "Code", ""]) + sheet.append([2, "Second link", "Chrome", "Global", "Missing destination", + "Open page", "No href", "Working link", "Add href", "2.4.4", "Code", ""]) + sheet.append([3, "Photo", "Chrome", "Home", "No alt text", "Open page", + "Image lacks alt", "Alt text", "Add alt", "1.1.1", "Code", ""]) + sheet.append([4, "Other", "Chrome", "Careers", "Outside scope"]) + workbook = tmp_path / "reference.xlsx" + book.save(workbook) + html = tmp_path / "home.html" + html.write_text('Home' + '

Home

Click here') + return workbook, html -def create_planned_run( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> Path: - """Plan a minimal evaluation run through the public CLI boundary.""" - monkeypatch.chdir(tmp_path) - workspace = PrivateWorkspace.from_current_directory() - workspace.initialize() - references = ReferenceSet( - version="synthetic-v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=(), - ) - save_reference_set(workspace.root / "references" / "approved.json", references) - snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" - snapshot_directory.mkdir(parents=True) - snapshot = snapshot_directory / "source.html" - snapshot.write_text("synthetic", encoding="utf-8") - (snapshot_directory / "metadata.json").write_text( - json.dumps({"sha256": hashlib.sha256(snapshot.read_bytes()).hexdigest()}), - encoding="utf-8", - ) - assert evaluation_main(["audit", "--max-audit-cost-usd", "0.01"]) == 0 - capsys.readouterr() - return next((workspace.root / "runs").iterdir()) - - -@pytest.fixture -def synthetic_workbook(tmp_path: Path) -> Path: - """Create the minimal authorized workbook placeholder used by CLI tests.""" - workbook = tmp_path / "synthetic-reference.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - return workbook - - -def test_help_exposes_the_evaluation_workflows() -> None: - """The CLI makes every evaluation lifecycle workflow discoverable.""" - result = run_cli("--help") - - assert result.returncode == 0 - for workflow in ("prepare", "audit", "review", "report"): - assert workflow in result.stdout - - -def test_prepare_initializes_one_partitioned_private_workspace( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """Prepare creates all private artifact partitions around a local workbook.""" - workspace = tmp_path / ".model-evaluation" - - result = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - cwd=tmp_path, - ) - - assert result.returncode == 0, result.stderr - assert "Private evaluation workspace initialized" in result.stdout - assert str(synthetic_workbook.resolve()) in result.stdout - assert sorted(path.name for path in workspace.iterdir()) == [ - "references", - "reports", - "reviews", - "runs", - "snapshots", - ] - - -def test_prepare_anchors_default_private_paths_to_the_worktree_root( - tmp_path: Path, -) -> None: - """Invocation from a subdirectory still uses the root-level ignore rules.""" - worktree = tmp_path / "worktree" - (worktree / ".git").mkdir(parents=True) - (worktree / ".gitignore").write_text( - "/Pristine Accessibility Defect Report.xlsx\n/.model-evaluation/\n", - encoding="utf-8", - ) - workbook = worktree / "Pristine Accessibility Defect Report.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - nested_directory = worktree / "nested" - nested_directory.mkdir() - - result = run_cli("prepare", cwd=nested_directory) - - assert result.returncode == 0, result.stderr - assert (worktree / ".model-evaluation").is_dir() - assert not (nested_directory / ".model-evaluation").exists() - - -def test_prepare_explains_how_to_supply_a_missing_private_workbook( - tmp_path: Path, -) -> None: - """A missing workbook fails with authorized, local recovery instructions.""" - missing_workbook = tmp_path / "missing.xlsx" - workspace = tmp_path / ".model-evaluation" - - result = run_cli( - "prepare", - "--workbook", - str(missing_workbook), - cwd=tmp_path, - ) - - assert result.returncode == 2 - assert "Private source workbook not found" in result.stderr - assert "authorized project source" in result.stderr - assert "--workbook PATH" in result.stderr - assert not workspace.exists() - - -def test_prepare_rejects_an_unignored_workbook_inside_the_worktree( - tmp_path: Path, -) -> None: - """A workbook override cannot make private data visible to version control.""" - with TemporaryDirectory(dir=PROJECT_ROOT) as temporary_directory: - workbook = Path(temporary_directory) / "private-input.xlsx" - workbook.write_bytes(b"synthetic workbook placeholder") - - result = run_cli( - "prepare", - "--workbook", - str(workbook), - cwd=tmp_path, - ) - - assert result.returncode == 2 - assert "would be visible to version control" in result.stderr - assert "outside the repository" in result.stderr - - -def test_prepare_captures_exact_html_with_snapshot_provenance( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """Prepare freezes the HTTP body consumed by the audit pipeline.""" - with local_snapshot_server() as (source_url, server): - result = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - - assert result.returncode == 0, result.stderr - assert "Benchmark snapshot captured" in result.stdout - assert server.request_count == 1 - assert server.request_user_agents == ["Mozilla/5.0"] - - snapshot_directory = ( - tmp_path / ".model-evaluation" / "snapshots" / "pristine-homepage" - ) - html_path = snapshot_directory / "source.html" - metadata_path = snapshot_directory / "metadata.json" - assert html_path.read_bytes() == SNAPSHOT_BODY - - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - assert metadata["source_url"] == source_url - assert metadata["sha256"] == SNAPSHOT_SHA256 - assert metadata["byte_length"] == len(SNAPSHOT_BODY) - assert metadata["html_file"] == html_path.name - assert metadata["http"] == { - "content_length": str(len(SNAPSHOT_BODY)), - "content_type": "text/html; charset=utf-8", - "etag": '"synthetic-v1"', - "final_url": source_url, - "last_modified": "Wed, 16 Sep 2026 14:00:00 GMT", - "request_user_agent": "Mozilla/5.0", - "status_code": 200, - } - retrieved_at = datetime.fromisoformat(metadata["retrieved_at"]) - assert retrieved_at.tzinfo is not None - - -def test_prepare_reuses_a_verified_snapshot_without_refetching( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """An existing benchmark is verified in place and never fetched again.""" - with local_snapshot_server() as (source_url, server): - first = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - second = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - - assert first.returncode == 0, first.stderr - assert second.returncode == 0, second.stderr - assert "Benchmark snapshot reused" in second.stdout - assert server.request_count == 1 - - -def test_prepare_rejects_html_the_audit_pipeline_cannot_decode( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """A snapshot must satisfy the pipeline's UTF-8 file contract.""" - with local_snapshot_server() as (source_url, _server): - result = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url.replace("/pristine-homepage", "/non-utf8"), - cwd=tmp_path, - ) - - assert result.returncode == 2 - assert "not valid UTF-8" in result.stderr - snapshot_directory = tmp_path / ".model-evaluation" / "snapshots" - assert list(snapshot_directory.iterdir()) == [] - - -def test_prepare_refuses_to_overwrite_a_changed_snapshot( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """Checksum drift fails closed without a network request or overwrite.""" - with local_snapshot_server() as (source_url, server): - first = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - assert first.returncode == 0, first.stderr - html_path = ( - tmp_path - / ".model-evaluation" - / "snapshots" - / "pristine-homepage" - / "source.html" - ) - html_path.write_bytes(b"changed benchmark") - - second = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - - assert second.returncode == 2 - assert "checksum does not match" in second.stderr - assert "will not be refetched or overwritten" in second.stderr - assert html_path.read_bytes() == b"changed benchmark" - assert server.request_count == 1 - - -def test_prepare_publishes_the_snapshot_as_one_immutable_bundle( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """A competing snapshot cannot be mixed with a capture in progress.""" - snapshot_directory = ( - tmp_path / ".model-evaluation" / "snapshots" / "pristine-homepage" - ) - - def publish_competing_snapshot() -> None: - """Simulate another prepare process winning the publication race.""" - snapshot_directory.mkdir() - (snapshot_directory / "race-marker").write_text( - "winner", - encoding="utf-8", - ) - - with local_snapshot_server(publish_competing_snapshot) as ( - source_url, - _server, - ): - result = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - - assert result.returncode == 2 - assert "appeared during capture" in result.stderr - assert [path.name for path in snapshot_directory.iterdir()] == ["race-marker"] - - -def test_prepare_rejects_the_dat_vision_aid_fixture_as_a_snapshot( - tmp_path: Path, - synthetic_workbook: Path, -) -> None: - """The unrelated committed fixture cannot stand in for Pristine.""" - result = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - "--snapshot-url", - str(PROJECT_ROOT / "test_files" / "dat_visionaid_home.html"), - cwd=tmp_path, - ) - - assert result.returncode == 2 - assert "DAT Vision Aid fixture is not a Pristine benchmark input" in result.stderr - - -@pytest.mark.parametrize("api_key_name", ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]) -def test_audit_defaults_to_a_network_free_dry_run_even_with_an_api_key( - tmp_path: Path, - synthetic_workbook: Path, - api_key_name: str, -) -> None: - """Environment credentials alone can never make evaluation billable.""" - prepared = run_cli( - "prepare", - "--workbook", - str(synthetic_workbook), - cwd=tmp_path, - ) - assert prepared.returncode == 0, prepared.stderr - - network_guard = tmp_path / "network_guard" - network_guard.mkdir() - (network_guard / "sitecustomize.py").write_text( - "import socket\n" - "def blocked_socket(*args, **kwargs):\n" - " raise AssertionError('evaluation CLI attempted network access')\n" - "socket.socket = blocked_socket\n", - encoding="utf-8", - ) - - result = run_cli( - "audit", - cwd=tmp_path, - env={ - api_key_name: "must-not-be-used", - "PYTHONPATH": str(network_guard), - }, - ) - - assert result.returncode == 0, result.stderr - assert "DRY RUN" in result.stdout - assert "No network requests were made" in result.stdout - - -@pytest.mark.parametrize("option", ("--plan", "--authorize", "--bundle")) -def test_obsolete_workflow_options_are_rejected(option: str) -> None: - """Removed plan, authorization, and report-bundle flags stay removed.""" - workflow = "report" if option == "--bundle" else "audit" - prefix = ("--run-dir", "obsolete-run") if workflow == "report" else () - result = run_cli(workflow, *prefix, option, "obsolete.json") - - assert result.returncode == 2 - assert "unrecognized arguments" in result.stderr - - -def test_interactive_live_approval_accepts_trimmed_case_insensitive_yes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """A terminal operator can approve the displayed run by typing yes.""" - run_directory = create_planned_run(tmp_path, monkeypatch, capsys) - class SyntheticClient: - """Return empty successful findings without provider access.""" +def args(workbook: Path, html: Path, *extra: str) -> list[str]: + return ["run", "--workbook", str(workbook), "--html", str(html), + "--source-url", "https://example.test/", "--max-cost-usd", "0.01", *extra] + - def __init__(self, **_kwargs: object) -> None: - pass +def directory(tmp_path: Path) -> Path: + return next((tmp_path / ".model-evaluation" / "runs").iterdir()) - def call(self, _prompt: str) -> dict[str, object]: - return { - "success": True, - "response": "[]", - "usage": {}, - "duration_seconds": 0, - } - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") - monkeypatch.setattr(sys, "stdin", TerminalInput(" YeS \n")) - monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) +def read_review(path: Path) -> list[dict[str, str]]: + with path.open(newline="") as stream: + return list(csv.DictReader(stream)) - assert evaluation_main(["audit", "--live", "--run-dir", str(run_directory)]) == 0 +def write_review(path: Path, rows: list[dict[str, str]]) -> None: + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=rows[0]) + writer.writeheader() + writer.writerows(rows) + + +def test_preview_preserves_reference_rows_and_never_calls_provider( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + workbook, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "available") + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: pytest.fail("paid call")) + assert main(args(workbook, html)) == 0 output = capsys.readouterr().out - manifest = json.loads( - (run_directory / "live-manifest.json").read_text(encoding="utf-8") - ) - assert "Evaluation execution summary" in output - assert "Approval mode: interactive" in output - assert manifest["approval"]["mode"] == "interactive" - - -@pytest.mark.parametrize( - ("response", "terminal", "message"), - ( - ("no\n", True, "was not 'yes'"), - ("\n", True, "was not 'yes'"), - ("", True, "ended before confirmation"), - ("yes\n", False, "requires a terminal"), - ), -) -def test_failed_live_approval_stops_before_client_or_artifacts( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - response: str, - terminal: bool, - message: str, + run = directory(tmp_path) + assert "claude-haiku-4-5-20251001" in output + assert "16000" in output and "24192" in output and "$0.01" in output + assert (run / "snapshot.html").read_bytes() == html.read_bytes() + manifest = json.loads((run / "run.json").read_text()) + assert manifest["snapshot_sha256"] == hashlib.sha256(html.read_bytes()).hexdigest() + rows = read_review(run / "review.csv") + assert [row["source_row"] for row in rows] == ["2", "3", "4"] + assert rows[0]["source_element"] == "First link" + assert rows[0]["actual result"] == "Vague link\nfor everyone" + assert all(not row["classification"] for row in rows) + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run)]) + + +def test_live_csv_review_and_report_full_row_metrics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Ambiguous, missing, or non-terminal input cannot begin a live run.""" - run_directory = create_planned_run(tmp_path, monkeypatch, capsys) - - class ForbiddenClient: - """Fail if approval rejection constructs a provider client.""" - - def __init__(self, **_kwargs: object) -> None: - raise AssertionError("provider client was constructed") - - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") - monkeypatch.setattr( - sys, - "stdin", - TerminalInput(response, terminal=terminal), - ) - monkeypatch.setattr(evaluation_audit, "AuditRequestClient", ForbiddenClient) - - with pytest.raises(SystemExit) as raised: - evaluation_main(["audit", "--live", "--run-dir", str(run_directory)]) - - assert raised.value.code == 2 - captured = capsys.readouterr() - assert "Evaluation execution summary" in captured.out - assert message in captured.err - for name in ( - "raw-attempts.json", - "canonical-audit-findings.json", - "live-manifest.json", - ): - assert not (run_directory / name).exists() - - -def test_live_option_combinations_keep_every_spending_gate_mandatory( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + workbook, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + calls = 0 + + def call(self, prompt: str) -> dict: + self.calls += 1 + return {"success": True, "response": json.dumps([{"problem": "Unclear link", + "location": "First link"}]), "usage": {"input_tokens": 100, + "output_tokens": 20, "cached_input_tokens": 50, + "cache_creation_input_tokens": 10, "reasoning_tokens": None}} + + fake = FakeClient() + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: fake) + assert main(args(workbook, html, "--live", "--approve-live")) == 0 + run = directory(tmp_path) + assert fake.calls == 3 + assert json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" + findings = json.loads((run / "audit-findings.json").read_text()) + programmatic = json.loads((run / "programmatic-findings.json").read_text()) + assert findings and programmatic + review = run / "review.csv" + rows = read_review(review) + rows[0].update(classification="llm_eligible", classification_reason="Model can see link", + audit_finding_id=findings[0]["finding_id"], match_reason="Full issue") + rows[1].update(classification="programmatic", classification_reason="HTML check", + programmatic_finding_id=programmatic[0]["finding_id"], match_reason="Full issue") + rows[2].update(classification="unavailable_evidence", classification_reason="Visual context") + write_review(review, rows) + assert main(["report", "--run-dir", str(run)]) == 0 + report = (run / "report.md").read_text() + assert "Workbook-row recall: 1/1" in report + assert "Programmatic coverage: 1/1" in report + assert "Combined workbook coverage: 2/3" in report + assert "Vague link" in report and "First link" in report + + +def test_review_requires_classification_and_rejects_invalid_finding_ids( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """Automation cannot blur planning, live intent, credentials, or cost.""" - run_directory = create_planned_run(tmp_path, monkeypatch, capsys) - workspace = PrivateWorkspace.from_evaluation_run(run_directory) - existing_runs = {path.name for path in (workspace.root / "runs").iterdir()} - + workbook, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + def call(self, prompt: str) -> dict: + return {"success": True, "response": json.dumps([{"problem": "Vague link"}]), + "usage": {"input_tokens": 1, "output_tokens": 1}} + + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: FakeClient()) + assert main(args(workbook, html, "--live", "--approve-live")) == 0 + run = directory(tmp_path) + review = run / "review.csv" + audit_ids = [item["finding_id"] for item in json.loads( + (run / "audit-findings.json").read_text())] + programmatic_id = json.loads( + (run / "programmatic-findings.json").read_text())[0]["finding_id"] + rows = read_review(review) with pytest.raises(SystemExit): - evaluation_main(["audit", "--max-audit-cost-usd", "0"]) - assert "greater than zero" in capsys.readouterr().err - assert {path.name for path in (workspace.root / "runs").iterdir()} == existing_runs - + main(["report", "--run-dir", str(run)]) + rows[0].update(classification="llm_eligible", classification_reason="Visible", + audit_finding_id="; ".join(audit_ids[:2]), + programmatic_finding_id=programmatic_id, + match_reason="Together cover the row", review_notes="Partial alone") + rows[1].update(classification="programmatic", classification_reason="Markup") + rows[2].update(classification="ambiguous", classification_reason="Unclear claim") + write_review(review, rows) + assert main(["report", "--run-dir", str(run)]) == 0 + assert "Combined workbook coverage: 1/3" in (run / "report.md").read_text() + rows[1]["audit_finding_id"] = audit_ids[0] + rows[1]["match_reason"] = "Conflicts" + write_review(review, rows) with pytest.raises(SystemExit): - evaluation_main(["audit", "--auto-approve"]) - assert "only with --live" in capsys.readouterr().err - - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") + main(["report", "--run-dir", str(run)]) + assert "Conflicting audit finding ID" in capsys.readouterr().err + rows[1]["audit_finding_id"] = "finding-unknown" + write_review(review, rows) with pytest.raises(SystemExit): - evaluation_main( - [ - "audit", - "--live", - "--run-dir", - str(run_directory), - "--auto-approve", - "--max-audit-cost-usd", - "1", - ] - ) - assert "reuses the saved cost guardrail" in capsys.readouterr().err - - monkeypatch.delenv("ANTHROPIC_API_KEY") - monkeypatch.setenv("OPENAI_API_KEY", "wrong-provider-key") + main(["report", "--run-dir", str(run)]) + assert "Unknown audit finding ID" in capsys.readouterr().err + rows[1]["audit_finding_id"] = programmatic_id + write_review(review, rows) with pytest.raises(SystemExit): - evaluation_main( - [ - "audit", - "--live", - "--run-dir", - str(run_directory), - "--auto-approve", - ] - ) - assert "requires ANTHROPIC_API_KEY" in capsys.readouterr().err - assert not (run_directory / "live-manifest.json").exists() - - -def test_cli_completes_the_synthetic_private_workflow( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """Prepare, review, audit, and report compose without provider access.""" - workbook_path = tmp_path / "synthetic.xlsx" - workbook = openpyxl.Workbook() - source = workbook.active - source.title = "Defects" - source.append(["Page", "URL", "Problem", "Location", "WCAG"]) - source.append( - [ - "Home", - "https://example.test/", - "Hero alternative text is misleading", - "Hero image", - "1.1.1", - ] - ) - workbook.save(workbook_path) - workbook_sha256 = hashlib.sha256(workbook_path.read_bytes()).hexdigest() - - with local_snapshot_server() as (source_url, _server): - prepared = run_cli( - "prepare", - "--workbook", - str(workbook_path), - "--workbook-sha256", - workbook_sha256, - "--homepage-url", - "https://example.test/", - "--snapshot-url", - source_url, - cwd=tmp_path, - ) - assert prepared.returncode == 0, prepared.stderr - - workspace = tmp_path / ".model-evaluation" - eligibility_path = workspace / "reviews" / "eligibility.xlsx" - review = openpyxl.load_workbook(eligibility_path) - eligibility = review["Eligibility"] - columns = {cell.value: cell.column for cell in eligibility[1]} - for name, value in { - "classification": "llm_eligible", - "decision": "accepted", - "rationale": "The model receives the image context.", - "reviewer": "reviewer@example.test", - "confidence": 0.9, - "timestamp": "2026-09-19T12:00:00Z", - }.items(): - eligibility.cell(2, columns[name], value) - review.save(eligibility_path) - - reviewed = run_cli( - "review", - "--eligibility-workbook", - str(eligibility_path), - cwd=tmp_path, - ) - assert reviewed.returncode == 0, reviewed.stderr - - planned = run_cli( - "audit", - "--max-audit-cost-usd", - "0.01", - cwd=tmp_path, - ) - assert planned.returncode == 0, planned.stderr - assert "DRY RUN" in planned.stdout - assert "Evaluation execution summary" in planned.stdout - assert "Maximum audit cost: $0.01" in planned.stdout - assert "Evaluation run:" in planned.stdout - - approved_path = workspace / "references" / "approved.json" - approved = json.loads(approved_path.read_text(encoding="utf-8")) - run_directories = list((workspace / "runs").iterdir()) - assert len(run_directories) == 1 - run_directory = run_directories[0] - plan_path = run_directory / "evaluation-manifest.json" - run_manifest = json.loads(plan_path.read_text(encoding="utf-8")) - assert run_directory.name == run_manifest["run_id"] - assert run_manifest["maximum_audit_cost_usd"] == "0.01" - frozen_reference = run_directory / "reference-set.json" - assert frozen_reference.read_bytes() == approved_path.read_bytes() - first_plan_bytes = plan_path.read_bytes() - repeated_plan = run_cli( - "audit", - "--max-audit-cost-usd", - "0.01", - cwd=tmp_path, - ) - assert repeated_plan.returncode == 0, repeated_plan.stderr - repeated_directories = list((workspace / "runs").iterdir()) - assert len(repeated_directories) == 2 - assert len({path.name for path in repeated_directories}) == 2 - assert plan_path.read_bytes() == first_plan_bytes - - class SyntheticClient: - """Return deterministic successful responses without network access.""" - - def __init__(self, **_kwargs: object) -> None: - pass - - def call(self, _prompt: str) -> dict[str, object]: - return { - "success": True, - "response": json.dumps( - { - "problem": "Hero alternative text is misleading", - "element": "img", - "location": "Hero image", - "wcag": "1.1.1", - } - ), - "usage": {"input_tokens": 10, "output_tokens": 5}, - "duration_seconds": 0.01, - "stop_reason": "stop", - } + main(["report", "--run-dir", str(run)]) + assert "Unknown audit finding ID" in capsys.readouterr().err + rows[0]["audit_finding_id"] = "" + rows[0]["programmatic_finding_id"] = "" + rows[0]["match_reason"] = "" + rows[0]["classification"] = "ambiguous" + rows[1]["audit_finding_id"] = "" + rows[1]["match_reason"] = "" + rows[2]["classification"] = "unavailable_evidence" + write_review(review, rows) + assert main(["report", "--run-dir", str(run)]) == 0 + assert "Workbook-row recall: N/A (0 eligible rows)" in (run / "report.md").read_text() + assert "Combined workbook coverage: 0/3" in (run / "report.md").read_text() + + +def test_live_execution_needs_explicit_approval( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workbook, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "available") + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: pytest.fail("paid call")) + + class DeclinedInput: + def isatty(self) -> bool: + return False + + monkeypatch.setattr("sys.stdin", DeclinedInput()) + assert main(args(workbook, html, "--live")) == 0 + assert json.loads((directory(tmp_path) / "run.json").read_text())["complete"] is False + +@pytest.mark.parametrize("failure", [True, False]) +def test_failed_or_over_budget_runs_keep_usage_and_cannot_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool +) -> None: + workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-key") - monkeypatch.setattr(evaluation_audit, "AuditRequestClient", SyntheticClient) - assert ( - evaluation_main( - [ - "audit", - "--live", - "--run-dir", - str(run_directory), - "--auto-approve", - ] - ) - == 0 - ) - live_output = capsys.readouterr().out - for expected in ( - "Model: claude-haiku-4-5-20251001", - "Endpoint: messages", - "Thinking: enabled; budget tokens: 16000", - "Benchmark snapshot SHA-256:", - "Reference set:", - "Prompts (", - "Estimated input tokens:", - "Retry policy:", - "Maximum audit cost: $0.01", - f"Destination: {run_directory}", - "Approval mode: auto", - ): - assert expected in live_output - live_manifest = json.loads( - (run_directory / "live-manifest.json").read_text(encoding="utf-8") - ) - assert live_manifest["approval"]["mode"] == "auto" - assert "approved_at" in live_manifest["approval"] - assert "authorization_digest" not in live_manifest - assert "authorized_plan_digest" not in live_manifest - live_artifacts = { - name: (run_directory / name).read_bytes() - for name in ( - "raw-attempts.json", - "canonical-audit-findings.json", - "live-manifest.json", - ) - } - with pytest.raises(SystemExit) as occupied: - evaluation_main( - [ - "audit", - "--live", - "--run-dir", - str(run_directory), - "--auto-approve", - ] - ) - assert occupied.value.code == 2 - assert "fresh evaluation run" in capsys.readouterr().err - for name, original_bytes in live_artifacts.items(): - assert (run_directory / name).read_bytes() == original_bytes - - snapshot_path = workspace / "snapshots" / "pristine-homepage" / "source.html" - snapshot_path.unlink() - - finding_path = run_directory / "canonical-audit-findings.json" - review_directory = workspace / "reviews" / run_manifest["run_id"] - matches_workbook = review_directory / "audit-matches.xlsx" - generated = run_cli( - "review", - "--run-dir", - str(run_directory), - "--kind", - "audit", - cwd=tmp_path, - ) - assert generated.returncode == 0, generated.stderr - match_review = openpyxl.load_workbook(matches_workbook) - matches = match_review["Matches"] - match_columns = {cell.value: cell.column for cell in matches[1]} - for row in range(2, matches.max_row + 1): - values = { - "decision": "accepted" if row == 2 else "rejected", - "page_compatible": True, - "failure_compatible": True, - "location_compatible": True, - "rationale": "Same reviewed homepage evidence.", - "reviewer": "reviewer@example.test", - "confidence": 0.9, - "timestamp": "2026-09-19T12:30:00Z", - } - for name, value in values.items(): - matches.cell(row, match_columns[name], value) - match_review.save(matches_workbook) - imported = run_cli( - "review", - "--run-dir", - str(run_directory), - "--kind", - "audit", - "--import-matches", - cwd=tmp_path, - ) - assert imported.returncode == 0, imported.stderr - programmatic_generated = run_cli( - "review", - "--run-dir", - str(run_directory), - "--kind", - "programmatic", - cwd=tmp_path, - ) - assert programmatic_generated.returncode == 0, programmatic_generated.stderr - programmatic_workbook = review_directory / "programmatic-matches.xlsx" - programmatic_review = openpyxl.load_workbook(programmatic_workbook) - programmatic_matches = programmatic_review["Matches"] - programmatic_columns = {cell.value: cell.column for cell in programmatic_matches[1]} - for row in range(2, programmatic_matches.max_row + 1): - for name, value in { - "decision": "rejected", - "rationale": "The deterministic finding is not the same defect.", - "reviewer": "reviewer@example.test", - "confidence": 0.9, - "timestamp": "2026-09-19T12:35:00Z", - }.items(): - programmatic_matches.cell(row, programmatic_columns[name], value) - programmatic_review.save(programmatic_workbook) - programmatic_imported = run_cli( - "review", - "--run-dir", - str(run_directory), - "--kind", - "programmatic", - "--import-matches", - cwd=tmp_path, - ) - assert programmatic_imported.returncode == 0, programmatic_imported.stderr - assert (review_directory / "audit-match-decisions.json").is_file() - assert (review_directory / "programmatic-match-decisions.json").is_file() - live_manifest["format_failures"] = ["synthetic-format"] - (run_directory / "live-manifest.json").write_text( - json.dumps(live_manifest), - encoding="utf-8", - ) - - reported = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) - - assert reported.returncode == 0, reported.stderr - json_report = workspace / "reports" / f"{run_manifest['run_id']}.json" - assert json_report.is_file() - report_data = json.loads(json_report.read_text(encoding="utf-8")) - assert report_data["parse_failures"] == ["synthetic-format"] - - programmatic_decisions = review_directory / "programmatic-match-decisions.json" - decision_bytes = programmatic_decisions.read_bytes() - programmatic_decisions.unlink() - unresolved = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) - assert unresolved.returncode == 2 - assert "--kind programmatic" in unresolved.stderr - assert "--import-matches" in unresolved.stderr - programmatic_decisions.write_bytes(decision_bytes) - - findings = json.loads(finding_path.read_text(encoding="utf-8")) - findings[0]["run_id"] = "different-run" - finding_path.write_text(json.dumps(findings), encoding="utf-8") - rejected_finding = run_cli("report", "--run-dir", str(run_directory), cwd=tmp_path) - assert rejected_finding.returncode == 2 - assert "do not belong to the verified run" in rejected_finding.stderr - - findings[0]["run_id"] = run_manifest["run_id"] - finding_path.write_text(json.dumps(findings), encoding="utf-8") - approved["references"][0]["eligibility_review"]["rationale"] = "Changed" - frozen_reference.write_text(json.dumps(approved), encoding="utf-8") - rejected_eligibility = run_cli( - "report", "--run-dir", str(run_directory), cwd=tmp_path - ) - assert rejected_eligibility.returncode == 2 - assert "reference-set" in rejected_eligibility.stderr - - -def test_haiku_plan_freezes_thinking_and_pricing(tmp_path, monkeypatch, capsys): - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - monkeypatch.setattr(evaluation_cli, "_PROJECT_ENV_LOADED", True) - run = create_planned_run(tmp_path, monkeypatch, capsys) - plan = evaluation_audit.load_audit_plan(run) - assert plan["configuration"] == { - "model": "claude-haiku-4-5-20251001", - "provider": "anthropic", - "endpoint": "messages", - "thinking": {"type": "enabled", "budget_tokens": 16000}, - "temperature": None, - "max_output_tokens": 24192, - "include_summaries": False, - "execution": "sequential", - } - summary = evaluation_cli._render_execution_summary(plan, run, "dry-run") - for value in ( - "anthropic", - "claude-haiku-4-5-20251001", - "messages", - "16000", - "24192", - "omitted", - plan["pricing_identity"], - "$0.01", - ): - assert value in summary - assert not (run / "live-manifest.json").exists() - - -def test_historical_luna_plan_is_readable_but_cannot_execute( - tmp_path, monkeypatch, capsys -): - from vision_aid.evaluation.serialization import json_digest - - run = create_planned_run(tmp_path, monkeypatch, capsys) - path = run / "evaluation-manifest.json" - plan = json.loads(path.read_text()) - plan["configuration"] = { - "model": "gpt-5.6-luna", - "reasoning_effort": "medium", - "endpoint": "chat.completions", - "temperature": None, - "max_output_tokens": 8192, - "include_summaries": False, - "execution": "sequential", - } - plan.pop("plan_digest") - plan["plan_digest"] = json_digest(plan) - path.write_text(json.dumps(plan)) - assert ( - evaluation_audit.load_audit_plan(run)["configuration"]["model"] - == "gpt-5.6-luna" - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "unused") + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + calls = 0 + + def call(self, prompt: str) -> dict: + self.calls += 1 + return {"success": not failure, + "response": "[]" if not failure else None, + "error": "synthetic failure" if failure else None, + "usage": {"input_tokens": 100000, "output_tokens": 20}} + + fake = FakeClient() + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: fake) + assert main(args(workbook, html, "--live", "--approve-live")) == 1 + run = directory(tmp_path) + manifest = json.loads((run / "run.json").read_text()) + assert fake.calls == 1 + assert manifest["complete"] is False + assert manifest["usage"]["input_tokens"] == 100000 + assert float(manifest["estimated_cost_usd"]) > 0 + assert len(json.loads((run / "raw-responses.json").read_text())) == 1 with pytest.raises(SystemExit): - evaluation_main(["audit", "--live", "--auto-approve", "--run-dir", str(run)]) - assert "not the frozen Haiku POC" in capsys.readouterr().err - assert not (run / "live-manifest.json").exists() + main(["report", "--run-dir", str(run)]) diff --git a/tests/test_match_review.py b/tests/test_match_review.py deleted file mode 100644 index a96256c..0000000 --- a/tests/test_match_review.py +++ /dev/null @@ -1,100 +0,0 @@ -from pathlib import Path - -import openpyxl - -from vision_aid.evaluation.matching import ( - HumanMatchReviewer, - export_match_workbook, - generate_candidates, -) -from vision_aid.evaluation.schemas import CanonicalFinding, ReferenceDefect - - -def _reference(identifier: str, scope: str = "home") -> ReferenceDefect: - """Build one synthetic image reference defect.""" - return ReferenceDefect( - reference_id=identifier, - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - source_sheet="Defects", - source_row=2, - raw_evidence={"Problem": "Image problem"}, - canonical_url="https://example.test/", - page_scope=scope, # type: ignore[arg-type] - problem="Image has the wrong alternative text", - location="hero image", - wcag_evidence=("1.1.1",), - ) - - -def _finding( - identifier: str, page: str = "https://example.test/" -) -> CanonicalFinding: - """Build one synthetic canonical audit finding.""" - return CanonicalFinding( - finding_id=identifier, - source="audit", - run_id="run-1", - model="gpt-5.6-luna", - prompt="informative_alt_quality", - checklist="CL03", - page_url=page, - problem_family="informative_alt_quality", - problem="Alternative text does not describe the hero", - element="img", - location="hero image", - wcag_evidence=("1.1.1",), - raw_source={"problem": "Alternative text does not describe the hero"}, - parse_status="parsed", - ) - - -def test_candidate_generation_only_removes_hard_incompatibilities() -> None: - """Global evidence remains eligible while different Home pages do not.""" - home = _reference("home") - global_reference = _reference("global", "global") - same_page = _finding("same") - other_page = _finding("other", "https://example.test/contact") - - candidates = generate_candidates( - (home, global_reference), (same_page, other_page) - ) - - assert ("home", "same") in candidates - assert ("home", "other") not in candidates - assert ("global", "same") in candidates - assert ("global", "other") in candidates - - -def test_human_match_review_round_trips_side_by_side_evidence( - tmp_path: Path, -) -> None: - """The Matches workbook returns provenance-complete decisions.""" - references = (_reference("ref1"),) - findings = (_finding("finding1"),) - review_path = tmp_path / "matches.xlsx" - export_match_workbook(references, findings, review_path) - - workbook = openpyxl.load_workbook(review_path) - sheet = workbook["Matches"] - columns = {cell.value: cell.column for cell in sheet[1]} - values = { - "decision": "accepted", - "page_compatible": True, - "failure_compatible": True, - "location_compatible": True, - "rationale": "Same alt-text failure on the hero image.", - "reviewer": "reviewer@example.test", - "confidence": 0.9, - "timestamp": "2026-09-19T12:00:00Z", - } - for name, value in values.items(): - sheet.cell(2, columns[name], value) - workbook.save(review_path) - - decisions = HumanMatchReviewer(review_path).review(references, findings) - - assert len(decisions) == 1 - assert decisions[0].state == "accepted" - assert decisions[0].page_compatible is True - assert decisions[0].provenance.reviewer == "reviewer@example.test" diff --git a/tests/test_programmatic_evaluation.py b/tests/test_programmatic_evaluation.py deleted file mode 100644 index 9953043..0000000 --- a/tests/test_programmatic_evaluation.py +++ /dev/null @@ -1,30 +0,0 @@ -from pathlib import Path - -from vision_aid.evaluation.programmatic import run_programmatic_audit - - -def test_existing_programmatic_checks_are_adapted_without_duplicating_rules( - tmp_path: Path, -) -> None: - """The evaluation adapter calls all existing deterministic checkers.""" - html = tmp_path / "benchmark.html" - html.write_text( - "Synthetic" - "

One

Two

", - encoding="utf-8", - ) - - result = run_programmatic_audit( - html, - run_id="run-1", - page_url="https://example.test/", - ) - - rule_ids = {finding.problem_family for finding in result.findings} - assert "HEAD_002" in rule_ids - assert "NON_TEXT_001" in rule_ids - assert result.raw_findings - assert all( - finding.raw_source in result.raw_findings - for finding in result.findings - ) diff --git a/tests/test_reference_set.py b/tests/test_reference_set.py deleted file mode 100644 index b36b976..0000000 --- a/tests/test_reference_set.py +++ /dev/null @@ -1,187 +0,0 @@ -import hashlib -from pathlib import Path - -import openpyxl -import pytest - -from vision_aid.evaluation.references import ( - HumanEligibilityReviewer, - ReferenceSetError, - export_eligibility_workbook, - import_homepage_references, -) - - -def _write_workbook(path: Path) -> str: - """Write a synthetic multi-scope workbook and return its checksum.""" - workbook = openpyxl.Workbook() - sheet = workbook.active - sheet.title = "Defects" - sheet.append( - ["Page", "URL", "Problem", "Location", "WCAG", "Recommendation"] - ) - sheet.append( - [ - "Home", - "https://example.test/", - ( - "Decorative image has descriptive alt text\n" - "(second line retained)" - ), - "Hero image", - "1.1.1", - "Use empty alternative text", - ] - ) - sheet.append( - [ - "Global", - "All pages", - "Heading labels are unclear", - "Primary navigation", - "2.4.6", - "Describe the section", - ] - ) - sheet.append( - [ - "Contact", - "https://example.test/contact", - "Contact-only issue", - "Form", - "3.3.2", - "Add instructions", - ] - ) - workbook.save(path) - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def test_import_homepage_references_retains_every_home_and_global_source_row( - tmp_path: Path, -) -> None: - """Home and Global imports remain lossless and source-addressed.""" - workbook_path = tmp_path / "synthetic.xlsx" - checksum = _write_workbook(workbook_path) - - reference_set = import_homepage_references( - workbook_path, - expected_sha256=checksum, - homepage_url="https://example.test/", - version="synthetic-v1", - ) - - assert [ - (item.source_sheet, item.source_row) - for item in reference_set.references - ] == [ - ("Defects", 2), - ("Defects", 3), - ] - assert reference_set.references[0].raw_evidence["Problem"] == ( - "Decorative image has descriptive alt text\n(second line retained)" - ) - assert reference_set.references[1].page_scope == "global" - assert reference_set.references[1].canonical_url == "https://example.test/" - assert reference_set.workbook_sha256 == checksum - - with pytest.raises(ReferenceSetError, match="checksum drift"): - import_homepage_references( - workbook_path, - expected_sha256="0" * 64, - homepage_url="https://example.test/", - version="synthetic-v1", - ) - - -def test_import_homepage_references_maps_mandated_workbook_headers( - tmp_path: Path, -) -> None: - """The mandated workbook's field names populate canonical evidence.""" - workbook_path = tmp_path / "mandated-headers.xlsx" - workbook = openpyxl.Workbook() - sheet = workbook.active - sheet.title = "Defect report" - sheet.append( - [ - "Page name", - "Issue Title", - "element name", - "WCAG Sc", - "Recommendation for Fix", - ] - ) - sheet.append( - [ - "Home", - "No main landmark defined on the page", - "Region", - "1.3.1", - "Use a main element.", - ] - ) - workbook.save(workbook_path) - checksum = hashlib.sha256(workbook_path.read_bytes()).hexdigest() - - reference_set = import_homepage_references( - workbook_path, - expected_sha256=checksum, - homepage_url="https://example.test/", - version="synthetic-v1", - ) - - reference = reference_set.references[0] - assert reference.problem == "No main landmark defined on the page" - assert reference.location == "Region" - assert reference.wcag_evidence == ("1.3.1",) - assert reference.recommendation == "Use a main element." - - -def test_human_eligibility_review_round_trips_validated_provenance( - tmp_path: Path, -) -> None: - """Eligibility decisions validate and retain human provenance.""" - workbook_path = tmp_path / "synthetic.xlsx" - checksum = _write_workbook(workbook_path) - reference_set = import_homepage_references( - workbook_path, - expected_sha256=checksum, - homepage_url="https://example.test/", - version="synthetic-v1", - ) - review_path = tmp_path / "eligibility.xlsx" - export_eligibility_workbook(reference_set, review_path) - - review = openpyxl.load_workbook(review_path) - sheet = review["Eligibility"] - headings = {cell.value: cell.column for cell in sheet[1]} - decisions = [ - ("llm_eligible", "accepted", "Human judgment is required."), - ("programmatic", "accepted", "A deterministic rule owns this defect."), - ] - for row, (classification, state, rationale) in enumerate( - decisions, start=2 - ): - sheet.cell(row, headings["classification"], classification) - sheet.cell(row, headings["decision"], state) - sheet.cell(row, headings["rationale"], rationale) - sheet.cell(row, headings["reviewer"], "reviewer@example.test") - sheet.cell(row, headings["confidence"], 0.9) - sheet.cell(row, headings["timestamp"], "2026-09-19T12:00:00Z") - review.save(review_path) - - reviewed = HumanEligibilityReviewer(review_path).review(reference_set) - - assert [item.eligibility for item in reviewed.references] == [ - "llm_eligible", - "programmatic", - ] - assert reviewed.references[0].eligibility_review is not None - assert reviewed.references[0].eligibility_review.reviewer == ( - "reviewer@example.test" - ) - - sheet.cell(2, headings["classification"], "secret_fifth_state") - review.save(review_path) - with pytest.raises(ReferenceSetError, match="classification"): - HumanEligibilityReviewer(review_path).review(reference_set) diff --git a/tests/test_scoring_and_reporting.py b/tests/test_scoring_and_reporting.py deleted file mode 100644 index c68666f..0000000 --- a/tests/test_scoring_and_reporting.py +++ /dev/null @@ -1,374 +0,0 @@ -import csv -import json -from dataclasses import replace -from pathlib import Path - -import pytest - -from vision_aid.evaluation.matching import MatchValidationError -from vision_aid.evaluation.reporting import write_reports -from vision_aid.evaluation.schemas import ( - CanonicalFinding, - MatchDecision, - ReferenceDefect, - ReferenceSet, - ReviewProvenance, - RunMetadata, - Usage, -) -from vision_aid.evaluation.scoring import rank_scores, score_evaluation - -REVIEW = ReviewProvenance( - reviewer="reviewer@example.test", - rationale="Evidence describes the same failure at the same location.", - confidence=0.95, - timestamp="2026-09-19T12:00:00Z", -) - - -def _reference( - identifier: str, - eligibility: str, - *, - required_subdefects: tuple[str, ...] = (), -) -> ReferenceDefect: - """Build one approved synthetic reference defect.""" - return ReferenceDefect( - reference_id=identifier, - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - source_sheet="Defects", - source_row=int(identifier[-1]), - raw_evidence={"Problem": identifier}, - canonical_url="https://example.test/", - page_scope="home", - problem=identifier, - location="hero", - wcag_evidence=("1.1.1",), - required_subdefects=required_subdefects, - eligibility=eligibility, # type: ignore[arg-type] - eligibility_state="accepted", - eligibility_review=REVIEW, - ) - - -def _finding(identifier: str, source: str = "audit") -> CanonicalFinding: - """Build one synthetic canonical finding.""" - return CanonicalFinding( - finding_id=identifier, - source=source, # type: ignore[arg-type] - run_id="run-1", - model="gpt-5.6-luna" if source == "audit" else None, - prompt="decorative_verification", - checklist="CL03", - page_url="https://example.test/", - problem_family="nontext", - problem=identifier, - element="img", - location="hero", - wcag_evidence=("1.1.1",), - raw_source={"problem": identifier}, - parse_status="parsed", - ) - - -def _match(reference: str, finding: str, subdefect: str | None = None) -> MatchDecision: - """Build one accepted, evidence-compatible match decision.""" - return MatchDecision( - reference_id=reference, - finding_id=finding, - state="accepted", - provenance=REVIEW, - required_subdefect=subdefect, - page_compatible=True, - failure_compatible=True, - location_compatible=True, - ) - - -def test_deterministic_score_separates_coverage() -> None: - """Scoring separates coverage and requires every compound sub-defect.""" - references = ReferenceSet( - version="v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=( - _reference("ref1", "llm_eligible"), - _reference("ref2", "llm_eligible", required_subdefects=("a", "b")), - _reference("ref3", "programmatic"), - _reference("ref4", "unavailable_evidence"), - _reference("ref5", "ambiguous"), - ), - ) - audit_findings = ( - _finding("finding1"), - _finding("finding2"), - _finding("extra"), - ) - programmatic_findings = (_finding("program1", "programmatic"),) - run = RunMetadata( - run_id="run-1", - model="gpt-5.6-luna", - complete=True, - comparable_identity="benchmark-v1", - audit_cost_usd="0.001234", - evaluation_cost_usd="0.02", - usage=Usage(input_tokens=100, output_tokens=25, reasoning_tokens=5), - wall_time_seconds=4.5, - request_durations_seconds=(1.0, 2.0), - ) - - partial = score_evaluation( - references, - audit_findings, - programmatic_findings, - (_match("ref1", "finding1"), _match("ref2", "finding2", "a")), - (_match("ref3", "program1"),), - run, - parse_failures=("heading_structure",), - ) - complete = score_evaluation( - references, - audit_findings, - programmatic_findings, - ( - _match("ref1", "finding1"), - _match("ref2", "finding2", "a"), - _match("ref2", "extra", "b"), - ), - (_match("ref3", "program1"),), - run, - parse_failures=("heading_structure",), - ) - - assert partial.workbook_row_recall == { - "caught": 1, - "denominator": 2, - "rate": "0.5", - } - assert complete.workbook_row_recall == { - "caught": 2, - "denominator": 2, - "rate": "1", - } - assert complete.programmatic_coverage == { - "caught": 1, - "denominator": 1, - "rate": "1", - } - assert complete.combined_workbook_coverage == { - "caught": 3, - "denominator": 5, - "rate": "0.6", - } - assert complete.unmatched_audit_finding_ids == () - assert complete.parse_failures == ("heading_structure",) - assert complete.total_experiment_cost_usd == "0.021234" - assert ( - complete.to_dict() - == score_evaluation( - references, - audit_findings, - programmatic_findings, - ( - _match("ref1", "finding1"), - _match("ref2", "finding2", "a"), - _match("ref2", "extra", "b"), - ), - (_match("ref3", "program1"),), - run, - parse_failures=("heading_structure",), - ).to_dict() - ) - - -def test_match_conflicts_and_incomplete_runs_cannot_be_ranked() -> None: - """Conflicting matches and incomplete runs fail closed.""" - references = ReferenceSet( - version="v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=( - _reference("ref1", "llm_eligible"), - _reference("ref2", "llm_eligible"), - ), - ) - finding = _finding("finding1") - run = RunMetadata( - run_id="run-1", - model="gpt-5.6-luna", - complete=False, - comparable_identity="benchmark-v1", - audit_cost_usd="0.01", - ) - with pytest.raises(MatchValidationError, match="one-to-one"): - score_evaluation( - references, - (finding,), - (), - (_match("ref1", "finding1"), _match("ref2", "finding1")), - (), - run, - ) - - score = score_evaluation(references, (finding,), (), (), (), run) - with pytest.raises(ValueError, match="incomplete"): - rank_scores((score,)) - - -@pytest.mark.parametrize("reasoning_tokens", [0, None]) -def test_reports_are_projections_of_one_score_object( - tmp_path: Path, reasoning_tokens -) -> None: - """Every format exposes the same totals, latency, and row disposition.""" - references = ReferenceSet( - version="v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=(_reference("ref1", "llm_eligible"),), - ) - finding = _finding("finding1") - run = RunMetadata( - run_id="run-1", - model="gpt-5.6-luna", - complete=True, - comparable_identity="benchmark-v1", - audit_cost_usd="0.001", - usage=Usage(output_tokens=16002, reasoning_tokens=reasoning_tokens), - ) - score = score_evaluation( - references, - (finding,), - (), - (_match("ref1", "finding1"),), - (), - run, - ) - - paths = write_reports(score, tmp_path) - - json_report = json.loads(paths.json.read_text(encoding="utf-8")) - with paths.csv.open(newline="", encoding="utf-8") as stream: - csv_rows = list(csv.DictReader(stream)) - markdown = paths.markdown.read_text(encoding="utf-8") - assert json_report["usage"]["reasoning_tokens"] == reasoning_tokens - assert json_report["usage"]["output_tokens"] == 16002 - if reasoning_tokens is None: - assert "Reasoning tokens: unavailable" in markdown - assert json_report["workbook_row_recall"]["caught"] == 1 - assert json_report["request_duration_sum_seconds"] == 0 - assert csv_rows[0]["model_caught"] == "True" - assert csv_rows[0]["workbook_recall_caught"] == "1" - assert csv_rows[0]["request_duration_sum_seconds"] == "0" - assert "1/1 (100.00%)" in markdown - assert "Input tokens: 0" in markdown - assert "Request duration sum: 0s" in markdown - assert "finding1" in markdown - assert REVIEW.rationale in markdown - - -def test_equal_recall_is_ranked_only_by_unrounded_audit_cost() -> None: - """Operational metrics cannot displace cost as the recall tie-breaker.""" - references = ReferenceSet( - version="v1", - workbook_filename="synthetic.xlsx", - workbook_sha256="a" * 64, - homepage_url="https://example.test/", - references=(_reference("ref1", "llm_eligible"),), - ) - finding = _finding("finding1") - base_run = RunMetadata( - run_id="expensive", - model="model-expensive", - complete=True, - comparable_identity="benchmark-v1", - audit_cost_usd="0.0012349", - workbook_sha256="a" * 64, - snapshot_sha256="snapshot-v1", - reference_set_version="v1", - prompt_hashes_identity="prompts-v1", - parser_identity="parser-v1", - eligibility_identity="eligibility-v1", - configuration_identity="configuration-v1", - ) - expensive_finding = replace( - finding, - run_id=base_run.run_id, - model=base_run.model, - ) - expensive = score_evaluation( - references, - (expensive_finding,), - (), - (_match("ref1", "finding1"),), - (), - base_run, - ) - cheap = score_evaluation( - references, - (replace(finding, run_id="cheap", model="model-cheap"),), - (), - (_match("ref1", "finding1"),), - (), - replace( - base_run, - run_id="cheap", - model="model-cheap", - audit_cost_usd="0.0012341", - wall_time_seconds=999, - ), - ) - - assert [score.model for score in rank_scores((expensive, cheap))] == [ - "model-cheap", - "model-expensive", - ] - - incompatible = score_evaluation( - replace(references, version="v2"), - (replace(finding, run_id="incompatible", model="other"),), - (), - (_match("ref1", "finding1"),), - (), - replace( - base_run, - run_id="incompatible", - model="other", - reference_set_version="v2", - ), - ) - with pytest.raises(ValueError, match="incompatible benchmark"): - rank_scores((expensive, incompatible)) - - for field in ( - "snapshot_sha256", - "prompt_hashes_identity", - "parser_identity", - "eligibility_identity", - "configuration_identity", - ): - changed_run = replace( - base_run, - run_id=f"changed-{field}", - model="other", - **{field: "changed"}, - ) - changed = score_evaluation( - references, - ( - replace( - finding, - run_id=changed_run.run_id, - model=changed_run.model, - ), - ), - (), - (_match("ref1", "finding1"),), - (), - changed_run, - ) - with pytest.raises(ValueError, match="incompatible benchmark"): - rank_scores((expensive, changed)) diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index 5aa0afa..5897736 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -1,609 +1,93 @@ -"""Production-pipeline adapter and guarded evaluation audit execution.""" +"""Fixed Haiku request configuration and sequential paid execution.""" from __future__ import annotations -import hashlib import json -import subprocess -import time -from collections.abc import Callable from dataclasses import asdict -from datetime import UTC, datetime from decimal import Decimal from pathlib import Path -from typing import Any, Protocol +from typing import Any -from processing_scripts.llm_client.audit import ( - AuditRequestClient, - AuditRequestConfig, -) -from vision_aid.evaluation.normalization import ( - normalize_programmatic_findings, - normalize_prompt_response, -) +from processing_scripts.llm_client.audit import AuditRequestClient, AuditRequestConfig +from vision_aid.evaluation.normalization import normalize_prompt_response from vision_aid.evaluation.pricing import PriceSchedule -from vision_aid.evaluation.serialization import ( - json_digest, - load_reference_set, - reference_set_identity, - write_json, -) MODEL = "claude-haiku-4-5-20251001" REQUEST_CONFIG = AuditRequestConfig( - model=MODEL, - thinking_budget_tokens=16000, - temperature=None, - max_output_tokens=24192, + model=MODEL, thinking_budget_tokens=16000, + max_output_tokens=24192, temperature=None, ) -RETRYABLE_FAILURES = ("timeout", "rate_limit", "server_error") -BACKOFF_SECONDS = (1, 2) -RETRY_POLICY = { - "maximum_additional_attempts": 2, - "backoff_seconds": list(BACKOFF_SECONDS), - "retryable": list(RETRYABLE_FAILURES), +CONFIGURATION = { + "model": MODEL, "provider": "anthropic", "endpoint": "messages", + "thinking_budget_tokens": 16000, "max_output_tokens": 24192, + "temperature": "omitted", "summaries": "disabled", } -PLAN_FILENAME = "evaluation-manifest.json" -LIVE_OUTPUT_FILENAMES = ( - "raw-attempts.json", - "canonical-audit-findings.json", - "live-manifest.json", +USAGE_KEYS = ( + "input_tokens", "cached_input_tokens", "cache_creation_input_tokens", + "output_tokens", ) -FROZEN_CONFIGURATION = { - "model": MODEL, - "provider": "anthropic", - "thinking": {"type": "enabled", "budget_tokens": 16000}, - "endpoint": "messages", - "temperature": None, - "max_output_tokens": 24192, - "include_summaries": False, - "execution": "sequential", -} - - -class AuditExecutionError(ValueError): - """An audit gate, identity check, or execution policy failed.""" -class RequestClient(Protocol): - def call(self, prompt: str) -> dict[str, Any]: - """Call one prompt and return the production client result shape.""" +def write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8") -Pipeline = Callable[..., dict[str, Any]] - - -def _sha256(path: Path) -> str: - """Hash one artifact without transforming its bytes.""" - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _positive_cost(value: str) -> Decimal: - """Parse one strictly positive audit-cost guardrail.""" +def positive_cost(value: str) -> Decimal: try: - cost_limit = Decimal(value) + amount = Decimal(value) except Exception as error: - raise AuditExecutionError( - "The audit cost limit must be a decimal amount." - ) from error - if not cost_limit.is_finite() or cost_limit <= 0: - raise AuditExecutionError( - "The audit cost limit must be greater than zero." - ) - return cost_limit - - -def normalize_audit_cost(value: str) -> str: - """Validate and normalize one planned maximum audit cost.""" - return format(_positive_cost(value).normalize(), "f") - - -def _recorded_path(path: Path, run_directory: Path) -> str: - """Prefer a run-relative evidence path while supporting external inputs.""" - resolved = path.resolve() - try: - return str(resolved.relative_to(run_directory.resolve())) - except ValueError: - return str(resolved) - - -def _resolve_recorded_path(value: str, run_directory: Path) -> Path: - """Resolve one plan artifact path against its evaluation run.""" - path = Path(value) - return path if path.is_absolute() else run_directory / path - - -def _repository_state(repository: Path) -> dict[str, Any]: - """Return the source-control commit and dirty state when available.""" - try: - commit = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=repository, - capture_output=True, - text=True, - check=True, - ).stdout.strip() - status = subprocess.run( - ["git", "status", "--porcelain"], - cwd=repository, - capture_output=True, - text=True, - check=True, - ).stdout - return {"commit": commit, "dirty": bool(status.strip())} - except (OSError, subprocess.CalledProcessError): - return {"commit": None, "dirty": None} - - -def build_audit_plan( - snapshot_path: Path, - output_directory: Path, - *, - maximum_audit_cost_usd: str, - reference_set_path: Path, - workbook_sha256: str, - snapshot_sha256: str, - reference_set_version: str, - eligibility_identity: str, - homepage_url: str = "benchmark-homepage", - pipeline: Pipeline | None = None, - repository: Path | None = None, - schedule: PriceSchedule | None = None, - run_id: str | None = None, -) -> dict[str, Any]: - """Run the existing pipeline dry and freeze the billable intent.""" - normalized_cost = normalize_audit_cost(maximum_audit_cost_usd) - if _sha256(snapshot_path) != snapshot_sha256: - raise AuditExecutionError( - "Benchmark snapshot checksum drift detected before audit planning." - ) - if pipeline is None: - from entry_points.run_pipeline import run_pipeline - - pipeline = run_pipeline - output_directory.mkdir(parents=True, exist_ok=True) - production_manifest = pipeline( - html_path=str(snapshot_path), - output_dir=output_directory, - api_key=None, - model=MODEL, - dry_run=True, - include_summaries=False, - request_config=REQUEST_CONFIG, - ) - prompt_entries = production_manifest.get("prompts_dry_run", []) - prompts = [] - prompt_hashes: dict[str, str] = {} - for entry in prompt_entries: - name = str(entry["name"]) - prompt_path = output_directory / "prompts" / f"{name}.json" - prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) - prompt_text = str(prompt_data["prompt_text"]) - prompt_hashes[name] = hashlib.sha256(prompt_text.encode()).hexdigest() - prompts.append( - { - "name": name, - "checklist": entry["checklist"], - "wcag_criteria": entry.get("wcag_criteria", []), - "estimated_input_tokens": int(entry["prompt_tokens_est"]), - "path": str(prompt_path.relative_to(output_directory)), - } - ) - price_schedule = schedule or PriceSchedule.default() - parser_path = Path(__file__).with_name("normalization.py") - now = datetime.now(UTC).isoformat().replace("+00:00", "Z") - frozen_references = load_reference_set(reference_set_path) - if frozen_references.workbook_sha256 != workbook_sha256: - raise AuditExecutionError( - "Frozen reference-set workbook identity does not match the plan." - ) - if frozen_references.version != reference_set_version: - raise AuditExecutionError( - "Frozen reference-set version does not match the plan." - ) - if reference_set_identity(frozen_references) != eligibility_identity: - raise AuditExecutionError( - "Frozen eligibility decisions do not match the plan." - ) - resolved_run_id = run_id or output_directory.resolve().name - raw_programmatic_path = output_directory / "programmatic_findings.json" - raw_programmatic = json.loads( - raw_programmatic_path.read_text(encoding="utf-8") - ) - canonical_programmatic = normalize_programmatic_findings( - raw_programmatic, - run_id=resolved_run_id, - page_url=homepage_url, - ) - canonical_programmatic_path = ( - output_directory / "canonical-programmatic.json" - ) - write_json( - canonical_programmatic_path, - [asdict(item) for item in canonical_programmatic], - ) - plan: dict[str, Any] = { - "kind": "model-evaluation-audit-plan", - "created_at": now, - "run_id": resolved_run_id, - "homepage_url": homepage_url, - "workbook_sha256": workbook_sha256, - "snapshot_sha256": snapshot_sha256, - "reference_set_version": reference_set_version, - "eligibility_identity": eligibility_identity, - "configuration": FROZEN_CONFIGURATION, - "filters": production_manifest.get("pass1_filters_active", []), - "skipped_prompts": production_manifest.get("prompts_skipped", []), - "request_count": len(prompts), - "estimated_input_tokens": sum( - item["estimated_input_tokens"] for item in prompts - ), - "prompts": prompts, - "prompt_hashes": prompt_hashes, - "parser_hash": _sha256(parser_path), - "pricing_version": price_schedule.version, - "pricing_identity": price_schedule.identity, - "retry_policy": RETRY_POLICY, - "maximum_audit_cost_usd": normalized_cost, - "execution_destination": str(output_directory.resolve()), - "snapshot_evidence": { - "path": _recorded_path(snapshot_path, output_directory), - "sha256": snapshot_sha256, - }, - "reference_evidence": { - "path": _recorded_path(reference_set_path, output_directory), - "sha256": _sha256(reference_set_path), - }, - "repository": _repository_state(repository or Path.cwd()), - "programmatic_findings": { - "count": len(canonical_programmatic), - "path": str( - canonical_programmatic_path.relative_to(output_directory) - ), - "sha256": _sha256(canonical_programmatic_path), - }, - } - plan["benchmark_identity"] = json_digest( - { - key: plan[key] - for key in ( - "workbook_sha256", - "snapshot_sha256", - "reference_set_version", - "eligibility_identity", - "configuration", - "filters", - "prompt_hashes", - "parser_hash", - "pricing_identity", - ) - } - ) - plan["plan_digest"] = json_digest(plan) - write_json(output_directory / PLAN_FILENAME, plan) - return plan - - -def load_audit_plan(run_directory: Path) -> dict[str, Any]: - """Load one run's immutable plan and self-contained evidence.""" - run_directory = run_directory.resolve() - plan_path = run_directory / PLAN_FILENAME - plan = json.loads(plan_path.read_text(encoding="utf-8")) - recorded_digest = plan.get("plan_digest") - digest_input = dict(plan) - digest_input.pop("plan_digest", None) - if recorded_digest != json_digest(digest_input): - raise AuditExecutionError( - "The saved evaluation plan has changed since creation." - ) - if plan.get("kind") != "model-evaluation-audit-plan": - raise AuditExecutionError("The evaluation run has an invalid plan kind.") - if plan.get("run_id") != run_directory.name: - raise AuditExecutionError( - "The evaluation run identifier does not match its directory. " - "Create a fresh evaluation run instead of copying or renaming one." - ) - _positive_cost(str(plan.get("maximum_audit_cost_usd", ""))) - reference = plan.get("reference_evidence") or {} - reference_path = _resolve_recorded_path( - str(reference.get("path", "")), run_directory - ) - if ( - not reference_path.is_file() - or _sha256(reference_path) != reference.get("sha256") - ): - raise AuditExecutionError( - "Frozen reference-set bytes differ from the evaluation plan." - ) - reference_set = load_reference_set(reference_path) - if ( - reference_set.workbook_sha256 != plan.get("workbook_sha256") - or reference_set.version != plan.get("reference_set_version") - or reference_set_identity(reference_set) - != plan.get("eligibility_identity") - ): - raise AuditExecutionError( - "Frozen reference-set identities differ from the evaluation plan." - ) - programmatic = plan.get("programmatic_findings") or {} - programmatic_path = run_directory / str(programmatic.get("path", "")) - if ( - not programmatic_path.is_file() - or _sha256(programmatic_path) != programmatic.get("sha256") - ): - raise AuditExecutionError( - "Programmatic findings differ from the evaluation plan." - ) - return plan - - -def _verified_prompt_text( - run_directory: Path, - plan: dict[str, Any], - prompt: dict[str, Any], -) -> str: - """Load one prompt only when its saved content identity still matches.""" - name = str(prompt["name"]) - prompt_path = run_directory / str(prompt["path"]) - prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) - prompt_text = str(prompt_data["prompt_text"]) - if ( - hashlib.sha256(prompt_text.encode()).hexdigest() - != plan["prompt_hashes"].get(name) - ): - raise AuditExecutionError(f"Prompt content drift detected for {name}.") - return prompt_text - - -def load_verified_audit_plan( - run_directory: Path, - *, - schedule: PriceSchedule | None = None, -) -> dict[str, Any]: - """Revalidate every point-of-use identity before live execution.""" - run_directory = run_directory.resolve() - plan = load_audit_plan(run_directory) - if plan.get("execution_destination") != str(run_directory): - raise AuditExecutionError( - "The live destination differs from the planned evaluation run. " - "Create a fresh evaluation run instead of copying one." - ) - if plan.get("configuration") != FROZEN_CONFIGURATION: - raise AuditExecutionError( - "The planned configuration is not the frozen Haiku POC." - ) - price_schedule = schedule or PriceSchedule.default() - if plan.get("pricing_identity") != price_schedule.identity: - raise AuditExecutionError( - "The price schedule differs from the evaluation plan." - ) - parser_path = Path(__file__).with_name("normalization.py") - if plan.get("parser_hash") != _sha256(parser_path): - raise AuditExecutionError( - "The parser identity differs from the evaluation plan." - ) - snapshot = plan.get("snapshot_evidence") or {} - snapshot_path = _resolve_recorded_path( - str(snapshot.get("path", "")), run_directory - ) - if ( - not snapshot_path.is_file() - or _sha256(snapshot_path) != snapshot.get("sha256") - or snapshot.get("sha256") != plan.get("snapshot_sha256") - ): - raise AuditExecutionError( - "Benchmark snapshot identity differs from the evaluation plan." - ) - for prompt in plan.get("prompts", ()): - _verified_prompt_text(run_directory, plan, prompt) - return plan - - -def ensure_live_destination_available(run_directory: Path) -> None: - """Refuse to overwrite any evidence from an earlier live execution.""" - occupied = [ - run_directory / name - for name in LIVE_OUTPUT_FILENAMES - if (run_directory / name).exists() - ] - if occupied: - names = ", ".join(path.name for path in occupied) - raise AuditExecutionError( - f"Live output already exists ({names}). Create a fresh " - "evaluation run before executing again." - ) - - -def _failure_classification(result: dict[str, Any]) -> str: - """Classify a provider failure for the bounded retry policy.""" - failure = result.get("failure") or {} - status = failure.get("status_code") - failure_type = str(failure.get("type") or "").casefold() - if status == 429 or "ratelimit" in failure_type: - return "rate_limit" - if ( - status in {408, 409} - or "timeout" in failure_type - or "connection" in failure_type - ): - return "timeout" - if isinstance(status, int) and status >= 500: - return "server_error" - if ( - status in {401, 403} - or "auth" in failure_type - or "permission" in failure_type - ): - return "authentication" - if isinstance(status, int) and 400 <= status < 500: - return "invalid_request" - return "permanent" - - -def _empty_usage() -> dict[str, int | None]: - """Create the token-category accumulator used across attempts.""" - return { - "input_tokens": 0, - "cached_input_tokens": 0, - "cache_creation_input_tokens": 0, - "output_tokens": 0, - "reasoning_tokens": 0, - } - - -def _add_usage(total: dict[str, int | None], result: dict[str, Any]) -> None: - """Add API-reported categories from one possibly billed attempt.""" - usage = result.get("usage") or {} - for key in total: - value = usage.get(key, 0) - previous = total[key] - total[key] = ( - None if value is None or previous is None else previous + int(value) - ) - - -def execute_audit_run( - run_directory: Path, - *, - live: bool = False, - api_key: str | None, - approval_mode: str, - approved_at: str, - client: RequestClient | None = None, - schedule: PriceSchedule | None = None, - sleep: Callable[[float], None] = time.sleep, -) -> dict[str, Any]: - """Execute one verified evaluation run after point-of-use approval.""" - if not live: - raise AuditExecutionError( - "Live execution requires the explicit --live flag." - ) - if not api_key: - raise AuditExecutionError("Live execution requires ANTHROPIC_API_KEY.") - if approval_mode not in {"interactive", "auto"}: - raise AuditExecutionError( - "Live execution requires interactive or automatic approval." - ) - run_directory = run_directory.resolve() - ensure_live_destination_available(run_directory) - plan = load_verified_audit_plan(run_directory, schedule=schedule) - recorded_digest = plan["plan_digest"] - cost_limit = _positive_cost(plan["maximum_audit_cost_usd"]) - price_schedule = schedule or PriceSchedule.default() - request_client = client or AuditRequestClient( - api_key=api_key, - request_config=REQUEST_CONFIG, - ) - output_directory = run_directory - requests: list[dict[str, Any]] = [] - totals = _empty_usage() + raise ValueError("Cost guardrail must be a positive decimal") from error + if not amount.is_finite() or amount <= 0: + raise ValueError("Cost guardrail must be a positive decimal") + return amount + + +def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: + """Make each paid request once, saving reported usage and responses as received.""" + schedule = PriceSchedule.load(run / "pricing.json") + client = AuditRequestClient(api_key=api_key, request_config=REQUEST_CONFIG) + usage: dict[str, int] = {key: 0 for key in USAGE_KEYS} + responses: list[dict[str, Any]] = [] + findings: list[dict[str, Any]] = [] format_failures: list[str] = [] - canonical_findings: list[dict[str, Any]] = [] complete = True - incomplete_reason: str | None = None - started = time.monotonic() - raw_attempts_path = output_directory / "raw-attempts.json" - for prompt in plan["prompts"]: - current_cost = Decimal(price_schedule.estimate(MODEL, totals)) - if current_cost >= cost_limit: - complete = False - incomplete_reason = "audit_cost_limit_exhausted" - break - prompt_text = _verified_prompt_text(output_directory, plan, prompt) - name = str(prompt["name"]) - attempts = [] - final_result: dict[str, Any] | None = None - budget_exhausted = False - for attempt_index in range(3): - current_cost = Decimal(price_schedule.estimate(MODEL, totals)) - if current_cost >= cost_limit: - complete = False - incomplete_reason = "audit_cost_limit_exhausted" - budget_exhausted = True - break - result = request_client.call(prompt_text) - _add_usage(totals, result) - classification = ( - "success" - if result.get("success") - else _failure_classification(result) - ) - attempts.append( - { - "attempt": attempt_index + 1, - "classification": classification, - "result": result, - } - ) - final_result = result - write_json( - raw_attempts_path, - { - "plan_digest": recorded_digest, - "requests": [ - *requests, - {"name": name, "attempts": attempts}, - ], - "usage": totals, - }, - ) - if result.get("success"): - break - if classification not in RETRYABLE_FAILURES: - complete = False - incomplete_reason = classification - break - if attempt_index < 2: - sleep(BACKOFF_SECONDS[attempt_index]) - if budget_exhausted: - if attempts: - requests.append({"name": name, "attempts": attempts}) + reason = None + for prompt in manifest["prompts"]: + if Decimal(schedule.estimate(MODEL, usage)) >= Decimal(manifest["max_cost_usd"]): + complete, reason = False, "cost guardrail exhausted" break - assert final_result is not None - request_record = {"name": name, "attempts": attempts} - requests.append(request_record) - if not final_result.get("success"): - complete = False - incomplete_reason = ( - incomplete_reason or "transient_failures_exhausted" + name = prompt["name"] + prompt_text = json.loads((run / "prompts" / f"{name}.json").read_text())["prompt_text"] + result: dict[str, Any] + try: + result = client.call(prompt_text) + except Exception as error: + result = {"success": False, "response": None, "usage": {}, + "error": str(error)} + reported: dict[str, Any] = result.get("usage") or {} + for key in USAGE_KEYS: + usage[key] += int(reported.get(key) or 0) + responses.append({"prompt": name, "result": result}) + if result.get("success"): + normalized = normalize_prompt_response( + name, str(result.get("response") or ""), run_id=manifest["run_id"], + model=MODEL, page_url=manifest["source_url"], ) + findings.extend(asdict(finding) for finding in normalized.findings) + if normalized.parse_status == "malformed": + format_failures.append(name) + else: + complete, reason = False, str(result.get("error") or "request failed") + manifest.update(usage=usage, estimated_cost_usd=schedule.estimate(MODEL, usage), + complete=False, incomplete_reason=reason, + format_failures=format_failures) + write_json(run / "raw-responses.json", responses) + write_json(run / "audit-findings.json", findings) + write_json(run / "run.json", manifest) + if not complete: break - normalized = normalize_prompt_response( - name, - str(final_result.get("response") or ""), - run_id=str(plan["run_id"]), - model=MODEL, - page_url=str(plan["homepage_url"]), - ) - if normalized.parse_status == "malformed": - format_failures.append(name) - canonical_findings.extend(asdict(item) for item in normalized.findings) - manifest = { - **plan, - "kind": "model-evaluation-live-run", - "approval": { - "mode": approval_mode, - "approved_at": approved_at, - }, - "completed_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), - "complete": complete, - "rankable": complete, - "incomplete_reason": incomplete_reason, - "requests": requests, - "usage": totals, - "estimated_audit_cost_usd": price_schedule.estimate(MODEL, totals), - "wall_time_seconds": round(time.monotonic() - started, 6), - "request_durations_seconds": [ - attempt["result"].get("duration_seconds", 0) - for request in requests - for attempt in request["attempts"] - ], - "format_failures": format_failures, - "canonical_findings": canonical_findings, - } - write_json( - output_directory / "canonical-audit-findings.json", - canonical_findings, - ) - write_json(output_directory / "live-manifest.json", manifest) - return manifest + manifest.update(complete=complete, incomplete_reason=reason, + estimated_cost_usd=schedule.estimate(MODEL, usage)) + write_json(run / "run.json", manifest) + return complete diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index d2d1c8d..c5b436c 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -1,574 +1,156 @@ -"""Command-line shell for the private model-evaluation lifecycle.""" +"""Run one private Haiku evaluation, then score a human-edited CSV.""" + +from __future__ import annotations import argparse +import hashlib import json import os +import shutil import sys -from collections.abc import Callable, Sequence -from dataclasses import dataclass +import uuid +from collections.abc import Sequence +from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path from dotenv import load_dotenv -from vision_aid.evaluation.matching import ( - HumanMatchReviewer, - MatchValidationError, - export_match_workbook, -) -from vision_aid.evaluation.references import ( - HumanEligibilityReviewer, - ReferenceSetError, - export_eligibility_workbook, - import_homepage_references, -) -from vision_aid.evaluation.reporting import write_reports -from vision_aid.evaluation.scoring import score_evaluation -from vision_aid.evaluation.serialization import ( - load_findings, - load_match_decisions, - load_reference_set, - load_verified_run_metadata, - reference_set_identity, - save_match_decisions, - save_reference_set, -) -from vision_aid.evaluation.snapshot import prepare_benchmark_snapshot -from vision_aid.evaluation.workspace import ( - DEFAULT_WORKBOOK_FILENAME, - PrivateInputError, - PrivateWorkspace, - require_private_workbook, +from entry_points.run_pipeline import run_pipeline +from vision_aid.evaluation.audit import ( + CONFIGURATION, MODEL, REQUEST_CONFIG, execute, positive_cost, write_json, ) +from vision_aid.evaluation.normalization import normalize_programmatic_findings +from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule +from vision_aid.evaluation.references import import_homepage_references +from vision_aid.evaluation.review import create_review, reviewed_rows, write_report -CommandHandler = Callable[[argparse.Namespace], int] PROJECT_ROOT = Path(__file__).resolve().parents[2] -_PROJECT_ENV_LOADED = False - - -def _load_project_env() -> None: - """Load local evaluation credentials once per CLI process.""" - global _PROJECT_ENV_LOADED - if _PROJECT_ENV_LOADED: - return - load_dotenv(PROJECT_ROOT / ".env") - _PROJECT_ENV_LOADED = True - - -@dataclass(frozen=True) -class RequiredRunArtifact: - """One conventional report input and its optional recovery review kind.""" - - label: str - path: Path - review_kind: str | None = None +DEFAULT_WORKBOOK = PROJECT_ROOT / "Pristine Accessibility Defect Report.xlsx" def build_parser() -> argparse.ArgumentParser: - """Build the discoverable evaluation command hierarchy.""" - parser = argparse.ArgumentParser( - prog="visionaid-evaluate", - description=( - "Prepare, audit, review, and report private model evaluations " - "without committing private data or spending money by default." - ), - ) - workflows = parser.add_subparsers( - title="workflows", - dest="workflow", - required=True, - ) - - prepare = workflows.add_parser( - "prepare", - help="initialize private inputs and artifact directories", - ) - prepare.add_argument( - "--workbook", - type=Path, - default=Path(DEFAULT_WORKBOOK_FILENAME), - help=( - "authorized local source workbook " - f"(default: ./{DEFAULT_WORKBOOK_FILENAME})" - ), - ) - prepare.add_argument( - "--workbook-sha256", - help="authorized SHA-256 required to import homepage reference rows", - ) - prepare.add_argument( - "--homepage-url", - help=( - "canonical Pristine homepage URL used to select Home and Global " - "rows" - ), - ) - prepare.add_argument( - "--reference-set-version", - default="pristine-homepage-v1", - help="curator-assigned version for the imported reference set", - ) - prepare.add_argument( - "--snapshot-url", - help="HTTP(S) source URL for an immutable Pristine homepage capture", - ) - prepare.set_defaults(handler=_prepare) - - audit = workflows.add_parser( - "audit", - help="inspect the prepared audit plan in network-free dry-run mode", - ) - audit.add_argument( - "--live", action="store_true", help="enable billable execution" - ) - audit.add_argument( - "--auto-approve", - action="store_true", - help="deliberately approve live execution without a terminal prompt", - ) - audit.add_argument( - "--max-audit-cost-usd", help="required planning cost guardrail" - ) - audit.add_argument( - "--run-dir", type=Path, help="planned evaluation run to execute" - ) - audit.set_defaults(handler=_audit) - - review = workflows.add_parser( - "review", - help="work with private eligibility and match review artifacts", - ) - review.add_argument( - "--eligibility-workbook", - type=Path, - help="apply a completed Eligibility workbook", - ) - review.add_argument( - "--reference-set", type=Path, help="reference-set JSON input" - ) - review.add_argument( - "--findings", type=Path, help="canonical findings JSON" - ) - review.add_argument( - "--run-dir", type=Path, help="evaluation run being reviewed" - ) - review.add_argument( - "--kind", - choices=("audit", "programmatic"), - help="finding collection being reviewed", - ) - review.add_argument( - "--matches-workbook", - type=Path, - help="generate or apply a Matches workbook", - ) - review.add_argument( - "--import-matches", - action="store_true", - help=( - "import decisions from --matches-workbook instead of generating it" - ), - ) - review.add_argument( - "--match-decisions", - type=Path, - help="validated match-decision JSON output override", - ) - review.set_defaults(handler=_review) - - report = workflows.add_parser( - "report", - help="generate private evaluation reports", - ) - report.add_argument( - "--run-dir", type=Path, required=True, help="evaluation run to report" - ) - report.add_argument( - "--output-dir", type=Path, help="private report directory" - ) - report.set_defaults(handler=_report) + parser = argparse.ArgumentParser(prog="visionaid-evaluate") + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="preview or execute the fixed Haiku experiment") + run.add_argument("--workbook", type=Path, default=DEFAULT_WORKBOOK) + run.add_argument("--html", type=Path, required=True, help="local Benchmark HTML") + run.add_argument("--source-url", required=True, help="URL represented by the HTML") + run.add_argument("--max-cost-usd", required=True, help="positive between-request limit") + run.add_argument("--live", action="store_true", help="enable billable execution") + run.add_argument("--approve-live", action="store_true", + help="explicitly approve billable execution without a prompt") + report = commands.add_parser("report", help="score completed run and edited review.csv") + report.add_argument("--run-dir", type=Path, required=True) return parser -def _prepare(arguments: argparse.Namespace) -> int: - """Validate the private workbook before creating the workspace layout.""" - workbook = require_private_workbook(arguments.workbook) - workspace = PrivateWorkspace.from_current_directory() - workspace.initialize() - resolved_workspace = workspace.root.resolve() - print(f"Private evaluation workspace initialized at {resolved_workspace}") - print(f"Private source workbook: {workbook}") - if bool(arguments.workbook_sha256) != bool(arguments.homepage_url): - raise PrivateInputError( - "Reference import requires both --workbook-sha256 and " - "--homepage-url." - ) - if arguments.workbook_sha256: - reference_set = import_homepage_references( - workbook, - expected_sha256=arguments.workbook_sha256, - homepage_url=arguments.homepage_url, - version=arguments.reference_set_version, - ) - imported_path = workspace.root / "references" / "imported.json" - review_path = workspace.root / "reviews" / "eligibility.xlsx" - save_reference_set(imported_path, reference_set) - export_eligibility_workbook(reference_set, review_path) - print(f"Imported reference set: {imported_path.resolve()}") - print(f"Eligibility review workbook: {review_path.resolve()}") - if arguments.snapshot_url: - snapshot = prepare_benchmark_snapshot( - workspace, - source_url=arguments.snapshot_url, - ) - action = "reused" if snapshot.reused else "captured" - print(f"Benchmark snapshot {action}: {snapshot.html_path.resolve()}") - print(f"Benchmark SHA-256: {snapshot.metadata.sha256}") - return 0 - - -def _render_execution_summary( - plan: dict[str, object], - run_directory: Path, - approval_mode: str, -) -> str: - """Render the complete billable intent for planning or live approval.""" - configuration = plan["configuration"] - assert isinstance(configuration, dict) - prompts = plan["prompts"] - assert isinstance(prompts, list) - prompt_names = ", ".join(str(item["name"]) for item in prompts) - retry_policy = plan["retry_policy"] - assert isinstance(retry_policy, dict) - return "\n".join( - ( - "Evaluation execution summary", - f" Model: {configuration['model']}", - f" Endpoint: {configuration['endpoint']}", - f" Provider: {configuration['provider']}", - f" Thinking: {configuration['thinking']['type']}; budget tokens: " - f"{configuration['thinking']['budget_tokens']}", - f" Total output cap: {configuration['max_output_tokens']}", - " Temperature: omitted", - f" Pricing: {plan['pricing_version']} ({plan['pricing_identity']})", - f" Benchmark snapshot SHA-256: {plan['snapshot_sha256']}", - f" Reference set: {plan['reference_set_version']}", - f" Reference identity: {plan['eligibility_identity']}", - f" Prompts ({plan['request_count']}): {prompt_names}", - f" Estimated input tokens: {plan['estimated_input_tokens']}", - " Retry policy: " - f"{retry_policy['maximum_additional_attempts']} additional " - f"attempts; backoff {retry_policy['backoff_seconds']}; " - f"retryable {retry_policy['retryable']}", - f" Maximum audit cost: ${plan['maximum_audit_cost_usd']}", - f" Destination: {run_directory.resolve()}", - f" Approval mode: {approval_mode}", - ) - ) - - -def _approval_timestamp() -> str: - """Return the UTC timestamp recorded for live-run approval.""" - return datetime.now(UTC).isoformat().replace("+00:00", "Z") - - -def _confirm_live_execution() -> str: - """Require exact, terminal-originated point-of-use confirmation.""" - if not sys.stdin.isatty(): - raise PrivateInputError( - "Interactive live-run approval requires a terminal. Use " - "--auto-approve only for deliberate automation." - ) - try: - response = input("Type 'yes' to begin billable execution: ") - except EOFError as error: - raise PrivateInputError( - "Live-run approval ended before confirmation; no request was made." - ) from error - if response.strip().casefold() != "yes": - raise PrivateInputError( - "Live-run approval was not 'yes'; no request was made." - ) - return _approval_timestamp() - - -def _audit(arguments: argparse.Namespace) -> int: - """Plan a no-cost evaluation run or execute one after explicit approval.""" - _load_project_env() - - from vision_aid.evaluation.audit import ( - build_audit_plan, - ensure_live_destination_available, - execute_audit_run, - load_verified_audit_plan, - normalize_audit_cost, - ) - - if arguments.auto_approve and not arguments.live: - raise PrivateInputError("--auto-approve is accepted only with --live.") - if arguments.live: - if arguments.run_dir is None: - raise PrivateInputError("--live requires --run-dir from planning.") - if arguments.max_audit_cost_usd is not None: - raise PrivateInputError( - "Live execution reuses the saved cost guardrail; do not pass " - "--max-audit-cost-usd." - ) - run_directory = arguments.run_dir.resolve() - plan = load_verified_audit_plan(run_directory) - ensure_live_destination_available(run_directory) - if not os.environ.get("ANTHROPIC_API_KEY"): - raise PrivateInputError( - "Live execution requires ANTHROPIC_API_KEY in addition to " - "approval." - ) - approval_mode = "auto" if arguments.auto_approve else "interactive" - print( - _render_execution_summary(plan, run_directory, approval_mode), - flush=True, - ) - approved_at = ( - _approval_timestamp() - if arguments.auto_approve - else _confirm_live_execution() - ) - manifest = execute_audit_run( - run_directory, - live=True, - api_key=os.environ.get("ANTHROPIC_API_KEY"), - approval_mode=approval_mode, - approved_at=approved_at, - ) - print(f"Live run complete: {manifest['complete']}") - print(f"Estimated audit cost: ${manifest['estimated_audit_cost_usd']}") - return 0 - if arguments.run_dir is not None: - raise PrivateInputError("--run-dir is accepted only with --live.") - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() - reference_path = workspace.root / "references" / "approved.json" - snapshot_directory = workspace.root / "snapshots" / "pristine-homepage" - snapshot_path = snapshot_directory / "source.html" - metadata_path = snapshot_directory / "metadata.json" - if ( - reference_path.is_file() - and snapshot_path.is_file() - and metadata_path.is_file() - ): - reference_set = load_reference_set(reference_path) - if any( - item.eligibility_state != "accepted" - for item in reference_set.references - ): - raise PrivateInputError( - "The reference set has unresolved eligibility decisions." - ) - snapshot_metadata = json.loads( - metadata_path.read_text(encoding="utf-8") - ) - if arguments.max_audit_cost_usd is None: - raise PrivateInputError( - "Audit planning requires --max-audit-cost-usd as a positive " - "spending guardrail." - ) - maximum_cost = normalize_audit_cost(arguments.max_audit_cost_usd) - eligibility_identity = reference_set_identity(reference_set) - run_id, run_directory = workspace.create_evaluation_run() - frozen_reference_path = run_directory / "reference-set.json" - save_reference_set(frozen_reference_path, reference_set) - plan = build_audit_plan( - snapshot_path, - run_directory, - maximum_audit_cost_usd=maximum_cost, - reference_set_path=frozen_reference_path, - workbook_sha256=reference_set.workbook_sha256, - snapshot_sha256=snapshot_metadata["sha256"], - reference_set_version=reference_set.version, - eligibility_identity=eligibility_identity, - homepage_url=reference_set.homepage_url, - run_id=run_id, - ) - print(f"DRY RUN: {plan['request_count']} sequential requests planned.") - print(_render_execution_summary(plan, run_directory, "pending")) - print(f"Evaluation run: {run_directory.resolve()}") +def _new_run_directory() -> Path: + root = Path.cwd() / ".model-evaluation" / "runs" + root.mkdir(parents=True, exist_ok=True) + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8] + run = root / run_id + run.mkdir() + return run + + +def _run(args: argparse.Namespace) -> int: + limit = positive_cost(args.max_cost_usd) + if args.approve_live and not args.live: + raise ValueError("--approve-live requires --live") + if not args.html.is_file(): + raise ValueError(f"Benchmark HTML not found: {args.html}") + references = import_homepage_references(args.workbook, args.source_url) + run = _new_run_directory() + shutil.copyfile(args.html, run / "snapshot.html") + shutil.copyfile(DEFAULT_SCHEDULE, run / "pricing.json") + snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() + write_json(run / "references.json", references.to_dict()) + production = run_pipeline( + html_path=str(run / "snapshot.html"), output_dir=run, api_key=None, + model=MODEL, dry_run=True, include_summaries=False, + request_config=REQUEST_CONFIG, + ) + (run / "manifest.json").unlink() + raw_programmatic = json.loads((run / "programmatic_findings.json").read_text()) + findings = normalize_programmatic_findings( + raw_programmatic, run_id=run.name, page_url=args.source_url, + ) + write_json(run / "programmatic-findings.json", [asdict(item) for item in findings]) + create_review(run / "review.csv", list(references.to_dict()["references"])) + schedule = PriceSchedule.load(run / "pricing.json") + manifest = { + "run_id": run.name, "source_url": args.source_url, + "workbook_filename": references.workbook_filename, + "workbook_sha256": references.workbook_sha256, + "snapshot_sha256": snapshot_hash, + "configuration": CONFIGURATION, + "pricing_version": schedule.version, "pricing_sha256": schedule.identity, + "max_cost_usd": str(limit), "complete": False, + "incomplete_reason": "preview", "estimated_cost_usd": "0", + "usage": {key: 0 for key in ( + "input_tokens", "cached_input_tokens", + "cache_creation_input_tokens", "output_tokens", + )}, + "format_failures": [], + "prompts": [{"name": entry["name"], "checklist": entry["checklist"]} + for entry in production["prompts_dry_run"]], + "filters": production["pass1_filters_active"], + "skipped_prompts": production["prompts_skipped"], + } + write_json(run / "run.json", manifest) + print(f"Evaluation run: {run}") + print(f"Model: {MODEL}; endpoint: Messages; thinking: 16000; output cap: 24192") + print(f"Temperature: omitted; summaries: disabled; prompts: {len(manifest['prompts'])}") + print(f"Snapshot SHA-256: {snapshot_hash}; Reference rows: {len(references.references)}") + print(f"Pricing: {schedule.version} ({schedule.identity})") + print(f"Cost guardrail: ${limit}; checked before each request") + print(f"Review CSV: {run / 'review.csv'}") + if not args.live: + print("Preview complete; no paid requests made.") return 0 - print( - "DRY RUN: No network requests were made and no API usage was incurred." - ) - print(f"Private evaluation workspace: {workspace.root.resolve()}") - return 0 - - -def _review(arguments: argparse.Namespace) -> int: - """Expose the private review workflow shell.""" - if arguments.eligibility_workbook: - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() - reference_path = arguments.reference_set or ( - workspace.root / "references" / "imported.json" - ) - reviewed = HumanEligibilityReviewer( - arguments.eligibility_workbook - ).review(load_reference_set(reference_path)) - approved_path = workspace.root / "references" / "approved.json" - save_reference_set(approved_path, reviewed) - print(f"Validated reference set: {approved_path.resolve()}") - return 0 - match_workflow_requested = any( - ( - arguments.run_dir, - arguments.kind, - arguments.reference_set, - arguments.findings, - arguments.matches_workbook, - arguments.match_decisions, - arguments.import_matches, - ) - ) - if match_workflow_requested: - if arguments.run_dir is None or arguments.kind is None: - raise PrivateInputError( - "Match review requires --run-dir and --kind " - "(audit or programmatic)." - ) - from vision_aid.evaluation.audit import load_audit_plan - - run_directory = arguments.run_dir.resolve() - workspace = PrivateWorkspace.from_evaluation_run(run_directory) - plan = load_audit_plan(run_directory) - run_id = str(plan["run_id"]) - review_directory = workspace.root / "reviews" / run_id - reference_path = arguments.reference_set or ( - run_directory / "reference-set.json" - ) - findings_path = arguments.findings or ( - run_directory - / ( - "canonical-audit-findings.json" - if arguments.kind == "audit" - else "canonical-programmatic.json" - ) - ) - workbook_path = arguments.matches_workbook or ( - review_directory / f"{arguments.kind}-matches.xlsx" - ) - decision_path = arguments.match_decisions or ( - review_directory / f"{arguments.kind}-match-decisions.json" - ) - references = load_reference_set(reference_path).references - findings = load_findings(findings_path) - if arguments.import_matches: - decisions = HumanMatchReviewer(workbook_path).review( - references, - findings, - ) - save_match_decisions(decision_path, decisions) - print(f"Validated match decisions: {decision_path.resolve()}") - else: - export_match_workbook(references, findings, workbook_path) - print(f"Matches review workbook: {workbook_path.resolve()}") - return 0 - workspace = PrivateWorkspace.from_current_directory() - workspace.require_initialized() - print( - f"Private review workspace: {(workspace.root / 'reviews').resolve()}" - ) - return 0 - - -def _report(arguments: argparse.Namespace) -> int: - """Discover verified run evidence and write every report projection.""" - from vision_aid.evaluation.audit import load_audit_plan - - run_directory = arguments.run_dir.resolve() - workspace = PrivateWorkspace.from_evaluation_run(run_directory) - plan = load_audit_plan(run_directory) - run_id = str(plan["run_id"]) - review_directory = workspace.root / "reviews" / run_id - artifacts = ( - RequiredRunArtifact( - "frozen reference set", run_directory / "reference-set.json" - ), - RequiredRunArtifact( - "audit findings", run_directory / "canonical-audit-findings.json" - ), - RequiredRunArtifact( - "programmatic findings", - run_directory / "canonical-programmatic.json", - ), - RequiredRunArtifact( - "audit match decisions", - review_directory / "audit-match-decisions.json", - "audit", - ), - RequiredRunArtifact( - "programmatic match decisions", - review_directory / "programmatic-match-decisions.json", - "programmatic", - ), - RequiredRunArtifact( - "live manifest", run_directory / "live-manifest.json" - ), - ) - for artifact in artifacts: - if artifact.path.is_file(): - continue - if artifact.review_kind is not None: - raise PrivateInputError( - f"Missing {artifact.label} at {artifact.path}. Run " - f"'visionaid-evaluate review --run-dir {run_directory} " - f"--kind {artifact.review_kind}', complete the workbook, " - "then rerun with " - "--import-matches." - ) - raise PrivateInputError( - f"Missing {artifact.label} at {artifact.path}; complete the " - "evaluation run before reporting." - ) - inputs = {artifact.label: artifact.path for artifact in artifacts} - reference_set = load_reference_set(inputs["frozen reference set"]) - live_manifest = json.loads( - inputs["live manifest"].read_text(encoding="utf-8") - ) - if ( - live_manifest.get("kind") != "model-evaluation-live-run" - or live_manifest.get("run_id") != run_id - or live_manifest.get("plan_digest") != plan.get("plan_digest") - ): - raise PrivateInputError( - "The live manifest does not belong to the verified evaluation run." - ) - score = score_evaluation( - reference_set, - load_findings(inputs["audit findings"]), - load_findings(inputs["programmatic findings"]), - load_match_decisions(inputs["audit match decisions"]), - load_match_decisions(inputs["programmatic match decisions"]), - load_verified_run_metadata(inputs["live manifest"], reference_set), - parse_failures=tuple(live_manifest.get("format_failures", ())), - ) - output_directory = arguments.output_dir or workspace.root / "reports" - paths = write_reports(score, output_directory) - print(f"JSON report: {paths.json.resolve()}") - print(f"CSV report: {paths.csv.resolve()}") - print(f"Markdown report: {paths.markdown.resolve()}") + load_dotenv(PROJECT_ROOT / ".env") + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + raise ValueError("Live execution requires ANTHROPIC_API_KEY") + if not args.approve_live: + if not sys.stdin.isatty() or input("Type 'yes' to approve paid execution: ") != "yes": + print("Live execution declined; no paid requests made.") + return 0 + complete = execute(run, manifest, api_key) + print(f"Run {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 + + +def _report(args: argparse.Namespace) -> int: + run = args.run_dir.resolve() + manifest = json.loads((run / "run.json").read_text()) + if manifest["run_id"] != run.name: + raise ValueError("Run evidence does not belong to this directory") + if not manifest["complete"]: + raise ValueError("Incomplete Evaluation runs cannot produce a final report") + reference_set = json.loads((run / "references.json").read_text()) + if reference_set["workbook_sha256"] != manifest["workbook_sha256"]: + raise ValueError("Reference workbook identity differs from the run") + audit = json.loads((run / "audit-findings.json").read_text()) + programmatic = json.loads((run / "programmatic-findings.json").read_text()) + for kind, collection in (("audit", audit), ("programmatic", programmatic)): + if any(item["run_id"] != run.name or item["source"] != kind or + item["page_url"] != manifest["source_url"] for item in collection): + raise ValueError(f"{kind} findings do not belong to this run") + rows = reviewed_rows(run / "review.csv", reference_set["references"], + audit, programmatic) + write_report(run / "report.md", manifest, rows, audit, programmatic) + print(f"Markdown report: {run / 'report.md'}") return 0 def main(argv: Sequence[str] | None = None) -> int: - """Run the requested evaluation workflow and render actionable errors.""" parser = build_parser() - arguments = parser.parse_args(argv) - handler: CommandHandler = arguments.handler + args = parser.parse_args(argv) try: - return handler(arguments) - except ( - MatchValidationError, - PrivateInputError, - ReferenceSetError, - OSError, - ValueError, - ) as error: + return _run(args) if args.command == "run" else _report(args) + except (OSError, ValueError, KeyError) as error: parser.error(str(error)) diff --git a/vision_aid/evaluation/matching.py b/vision_aid/evaluation/matching.py deleted file mode 100644 index 43002cd..0000000 --- a/vision_aid/evaluation/matching.py +++ /dev/null @@ -1,404 +0,0 @@ -"""Candidate generation and reviewed one-to-one match validation.""" - -from __future__ import annotations - -import json -from collections import Counter -from collections.abc import Sequence -from pathlib import Path -from typing import Any, Protocol - -import openpyxl -from openpyxl.worksheet.datavalidation import DataValidation - -from vision_aid.evaluation.schemas import ( - CanonicalFinding, - MatchDecision, - ReferenceDefect, - ReviewProvenance, -) - - -class MatchValidationError(ValueError): - """Match decisions are unresolved, incompatible, or conflicting.""" - - -class MatchReviewer(Protocol): - """Adapter contract for human or future model match review.""" - - def review( - self, - references: Sequence[ReferenceDefect], - findings: Sequence[CanonicalFinding], - ) -> tuple[MatchDecision, ...]: - """Return validated decisions for candidate finding/reference pairs.""" - - -MATCH_COLUMNS = ( - "reference_id", - "finding_id", - "reference_scope", - "reference_problem", - "reference_location", - "reference_wcag", - "reference_raw_evidence", - "finding_prompt", - "finding_problem", - "finding_element", - "finding_location", - "finding_wcag", - "finding_raw_source", - "decision", - "page_compatible", - "failure_compatible", - "location_compatible", - "required_subdefect", - "rationale", - "reviewer", - "confidence", - "timestamp", - "notes", -) - - -def generate_candidates( - references: Sequence[ReferenceDefect], - findings: Sequence[CanonicalFinding], -) -> tuple[tuple[str, str], ...]: - """Remove only hard page/element incompatibilities; never award credit.""" - candidates: list[tuple[str, str]] = [] - for reference in references: - for finding in findings: - page_compatible = ( - reference.page_scope == "global" - or reference.canonical_url == finding.page_url - ) - if not page_compatible: - continue - reference_location = reference.location.casefold() - finding_evidence = ( - f"{finding.element} {finding.location}".casefold() - ) - if reference_location and finding_evidence: - # Only explicit, disjoint element categories are impossible. - categories = ( - "image", - "heading", - "link", - "form", - "table", - "media", - ) - ref_categories = { - item for item in categories if item in reference_location - } - finding_categories = { - item for item in categories if item in finding_evidence - } - if ( - ref_categories - and finding_categories - and ref_categories.isdisjoint(finding_categories) - ): - continue - candidates.append((reference.reference_id, finding.finding_id)) - return tuple(candidates) - - -def validate_match_decisions( - references: Sequence[ReferenceDefect], - findings: Sequence[CanonicalFinding], - decisions: Sequence[MatchDecision], - *, - require_resolved: bool = True, -) -> tuple[MatchDecision, ...]: - """Require resolved evidence-backed decisions and one-to-one findings.""" - reference_by_id = {item.reference_id: item for item in references} - finding_ids = {item.finding_id for item in findings} - pair_counts = Counter( - (item.reference_id, item.finding_id) for item in decisions - ) - repeated = [pair for pair, count in pair_counts.items() if count > 1] - if repeated: - raise MatchValidationError( - f"Match review repeats candidate pair {repeated[0]!r}." - ) - for decision in decisions: - if decision.reference_id not in reference_by_id: - raise MatchValidationError( - f"Unknown reference in match review: {decision.reference_id}." - ) - if decision.finding_id not in finding_ids: - raise MatchValidationError( - f"Unknown finding in match review: {decision.finding_id}." - ) - if require_resolved and decision.state == "needs_review": - raise MatchValidationError( - "Unresolved needs_review match decisions cannot be scored." - ) - if ( - not decision.provenance.reviewer - or not decision.provenance.rationale - ): - raise MatchValidationError( - "Every match decision requires provenance." - ) - if not 0 <= decision.provenance.confidence <= 1: - raise MatchValidationError("Match confidence must be 0 through 1.") - if decision.state != "accepted": - continue - if not all( - ( - decision.page_compatible, - decision.failure_compatible, - decision.location_compatible, - ) - ): - raise MatchValidationError( - "An accepted match requires compatible page, underlying " - "failure, " - "and location or element-group evidence." - ) - reference = reference_by_id[decision.reference_id] - if reference.required_subdefects: - if ( - decision.required_subdefect - not in reference.required_subdefects - ): - reference_id = reference.reference_id - raise MatchValidationError( - f"Accepted match for {reference_id} must identify " - "one " - "of its required sub-defects." - ) - elif decision.required_subdefect: - raise MatchValidationError( - f"Reference {reference.reference_id} has no required " - "sub-defects." - ) - accepted = [item for item in decisions if item.state == "accepted"] - finding_counts = Counter(item.finding_id for item in accepted) - conflicts = [item for item, count in finding_counts.items() if count > 1] - if conflicts: - raise MatchValidationError( - "Accepted matches must be one-to-one; finding " - f"{conflicts[0]} claims multiple reference defects." - ) - plain_reference_counts = Counter( - item.reference_id - for item in accepted - if not reference_by_id[item.reference_id].required_subdefects - ) - conflicts = [ - item for item, count in plain_reference_counts.items() if count > 1 - ] - if conflicts: - raise MatchValidationError( - "Accepted matches must be one-to-one; reference " - f"{conflicts[0]} is claimed by multiple findings." - ) - subdefect_counts = Counter( - (item.reference_id, item.required_subdefect) - for item in accepted - if item.required_subdefect is not None - ) - subdefect_conflicts = [ - item for item, count in subdefect_counts.items() if count > 1 - ] - if subdefect_conflicts: - raise MatchValidationError( - "Accepted matches must be one-to-one at the required " - "sub-defect level." - ) - return tuple(decisions) - - -def export_match_workbook( - references: Sequence[ReferenceDefect], - findings: Sequence[CanonicalFinding], - output_path: Path, -) -> None: - """Create an evidence-rich private review workbook for plausible pairs.""" - reference_by_id = {item.reference_id: item for item in references} - finding_by_id = {item.finding_id: item for item in findings} - workbook = openpyxl.Workbook() - sheet = workbook.active - sheet.title = "Matches" - sheet.append(MATCH_COLUMNS) - for reference_id, finding_id in generate_candidates(references, findings): - reference = reference_by_id[reference_id] - finding = finding_by_id[finding_id] - sheet.append( - ( - reference_id, - finding_id, - reference.page_scope, - reference.problem, - reference.location, - ", ".join(reference.wcag_evidence), - json.dumps( - reference.raw_evidence, ensure_ascii=False, sort_keys=True - ), - finding.prompt, - finding.problem, - finding.element, - finding.location, - ", ".join(finding.wcag_evidence), - json.dumps( - finding.raw_source, ensure_ascii=False, sort_keys=True - ), - "needs_review", - "", - "", - "", - "", - "", - "", - "", - "", - "", - ) - ) - sheet.freeze_panes = "A2" - sheet.auto_filter.ref = sheet.dimensions - for column in ("D", "E", "G", "I", "K", "M", "S", "W"): - sheet.column_dimensions[column].width = 40 - for row in sheet.iter_rows(min_row=2): - for cell in row: - cell.alignment = openpyxl.styles.Alignment( - wrap_text=True, - vertical="top", - ) - decisions = DataValidation( - type="list", - formula1='"accepted,rejected,needs_review"', - ) - booleans = DataValidation(type="list", formula1='"TRUE,FALSE"') - sheet.add_data_validation(decisions) - sheet.add_data_validation(booleans) - decisions.add(f"N2:N{max(2, sheet.max_row)}") - booleans.add(f"O2:Q{max(2, sheet.max_row)}") - output_path.parent.mkdir(parents=True, exist_ok=True) - workbook.save(output_path) - - -def _text(value: Any) -> str: - """Normalize an optional worksheet cell to stripped text.""" - return "" if value is None else str(value).strip() - - -def _boolean(value: Any, *, row: int, field: str) -> bool | None: - """Parse an optional human-entered worksheet boolean.""" - if value in (None, ""): - return None - if isinstance(value, bool): - return value - normalized = str(value).strip().casefold() - if normalized in {"true", "yes", "1"}: - return True - if normalized in {"false", "no", "0"}: - return False - raise MatchValidationError(f"Match row {row} has invalid {field} value.") - - -class HumanMatchReviewer: - """Load human match decisions through the shared reviewer contract.""" - - def __init__(self, review_path: Path): - """Select the private Matches workbook to read.""" - self.review_path = review_path - - def review( - self, - references: Sequence[ReferenceDefect], - findings: Sequence[CanonicalFinding], - ) -> tuple[MatchDecision, ...]: - """Load complete decisions and validate accepted conflicts.""" - try: - workbook = openpyxl.load_workbook( - self.review_path, - read_only=True, - data_only=True, - ) - sheet = workbook["Matches"] - except (OSError, KeyError, ValueError) as error: - raise MatchValidationError( - f"Could not read Matches review workbook: {error}" - ) from error - rows = sheet.iter_rows(values_only=True) - header_values = next(rows, None) or () - headers = [_text(value) for value in header_values] - missing = set(MATCH_COLUMNS) - set(headers) - if missing: - raise MatchValidationError( - "Matches review is missing columns: " - + ", ".join(sorted(missing)) - ) - decisions = [] - for row_number, values in enumerate(rows, start=2): - record = dict(zip(headers, values, strict=False)) - reference_id = _text(record.get("reference_id")) - finding_id = _text(record.get("finding_id")) - if not reference_id and not finding_id: - continue - state = _text(record.get("decision")) - if state not in {"accepted", "rejected", "needs_review"}: - raise MatchValidationError( - f"Match row {row_number} has invalid decision {state!r}." - ) - reviewer = _text(record.get("reviewer")) - rationale = _text(record.get("rationale")) - timestamp = _text(record.get("timestamp")) - if not reviewer or not rationale or not timestamp: - raise MatchValidationError( - f"Match row {row_number} requires reviewer, rationale, " - "and timestamp." - ) - try: - raw_confidence = record.get("confidence") - if raw_confidence is None: - raise ValueError - confidence = float(raw_confidence) - except (TypeError, ValueError) as error: - raise MatchValidationError( - f"Match row {row_number} requires numeric confidence." - ) from error - decisions.append( - MatchDecision( - reference_id=reference_id, - finding_id=finding_id, - state=state, # type: ignore[arg-type] - provenance=ReviewProvenance( - reviewer=reviewer, - rationale=rationale, - confidence=confidence, - timestamp=timestamp, - ), - required_subdefect=( - _text(record.get("required_subdefect")) or None - ), - page_compatible=_boolean( - record.get("page_compatible"), - row=row_number, - field="page_compatible", - ), - failure_compatible=_boolean( - record.get("failure_compatible"), - row=row_number, - field="failure_compatible", - ), - location_compatible=_boolean( - record.get("location_compatible"), - row=row_number, - field="location_compatible", - ), - notes=_text(record.get("notes")), - ) - ) - workbook.close() - return validate_match_decisions( - references, - findings, - decisions, - require_resolved=False, - ) diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index 42e7531..2ca36c9 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -56,27 +56,14 @@ def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: usage.get("cache_creation_input_tokens", 0) or 0 ) output_tokens = Decimal(usage.get("output_tokens", 0) or 0) - uncached_tokens = max(Decimal(0), input_tokens - cached_tokens) - input_multiplier = Decimal(1) - output_multiplier = Decimal(1) - threshold = Decimal(rates.get("long_context_threshold_tokens", 0)) - if threshold and input_tokens > threshold: - input_multiplier = Decimal(rates["long_context_input_multiplier"]) - output_multiplier = Decimal( - rates["long_context_output_multiplier"] - ) cost = ( - uncached_tokens + input_tokens * Decimal(rates["input_per_million_usd"]) - * input_multiplier + cached_tokens * Decimal(rates["cached_input_per_million_usd"]) - * input_multiplier + cache_creation_tokens * Decimal(rates["cache_creation_input_per_million_usd"]) - * input_multiplier + output_tokens * Decimal(rates["output_per_million_usd"]) - * output_multiplier ) / MILLION return format(cost.normalize(), "f") diff --git a/vision_aid/evaluation/pricing.v1.json b/vision_aid/evaluation/pricing.v1.json deleted file mode 100644 index 2a33d66..0000000 --- a/vision_aid/evaluation/pricing.v1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models": { - "gpt-5.6-luna": { - "cache_creation_input_per_million_usd": "0.25", - "cached_input_per_million_usd": "0.02", - "input_per_million_usd": "0.20", - "long_context_input_multiplier": "2", - "long_context_output_multiplier": "1.5", - "long_context_threshold_tokens": 272000, - "output_per_million_usd": "1.20" - } - }, - "published_at": "2026-07-09", - "source": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", - "version": "openai-public-2026-07-09" -} diff --git a/vision_aid/evaluation/pricing.v2.json b/vision_aid/evaluation/pricing.v2.json index e15bca5..6e7ed53 100644 --- a/vision_aid/evaluation/pricing.v2.json +++ b/vision_aid/evaluation/pricing.v2.json @@ -1,16 +1,5 @@ { "models": { - "gpt-5.6-luna": { - "cache_creation_input_per_million_usd": "0.25", - "cached_input_per_million_usd": "0.02", - "input_per_million_usd": "0.20", - "long_context_input_multiplier": "2", - "long_context_output_multiplier": "1.5", - "long_context_threshold_tokens": 272000, - "output_per_million_usd": "1.20", - "source": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", - "published_at": "2026-07-09" - }, "claude-haiku-4-5-20251001": { "input_per_million_usd": "1", "output_per_million_usd": "5", diff --git a/vision_aid/evaluation/programmatic.py b/vision_aid/evaluation/programmatic.py deleted file mode 100644 index 9d87e73..0000000 --- a/vision_aid/evaluation/programmatic.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Adapter around the production programmatic accessibility checks.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from processing_scripts.programmatic.forms_checklist_02 import audit_forms -from processing_scripts.programmatic.nontext_checklist_03 import audit_nontext -from processing_scripts.programmatic.semantic_checklist_01 import ( - audit_html_file, -) -from vision_aid.evaluation.normalization import normalize_programmatic_findings -from vision_aid.evaluation.schemas import CanonicalFinding - - -@dataclass(frozen=True) -class ProgrammaticAudit: - """Raw checker evidence and its canonical comparison records.""" - - raw_findings: tuple[dict[str, Any], ...] - findings: tuple[CanonicalFinding, ...] - - -def run_programmatic_audit( - html_path: Path, - *, - run_id: str, - page_url: str, -) -> ProgrammaticAudit: - """Run the existing three checker entry points against a benchmark file.""" - path = str(html_path) - raw = tuple( - audit_html_file(path) + audit_forms(path) + audit_nontext(path) - ) - return ProgrammaticAudit( - raw_findings=raw, - findings=normalize_programmatic_findings( - raw, - run_id=run_id, - page_url=page_url, - ), - ) diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/references.py index 602f30e..fb4aeaf 100644 --- a/vision_aid/evaluation/references.py +++ b/vision_aid/evaluation/references.py @@ -1,413 +1,59 @@ -"""Import workbook rows into traceable homepage reference defects.""" +"""Import the known Reference workbook's Home and Global defect rows.""" from __future__ import annotations import hashlib -import json import re -from dataclasses import replace from pathlib import Path -from typing import Literal, Protocol -from urllib.parse import urlsplit, urlunsplit import openpyxl -from openpyxl.worksheet.datavalidation import DataValidation - -from vision_aid.evaluation.schemas import ( - ReferenceDefect, - ReferenceSet, - ReviewProvenance, -) - - -class ReferenceSetError(ValueError): - """The private workbook cannot produce a trustworthy reference set.""" +from vision_aid.evaluation.schemas import ReferenceDefect, ReferenceSet -class EligibilityReviewer(Protocol): - """Adapter contract for human or future model eligibility review.""" - - def review(self, reference_set: ReferenceSet) -> ReferenceSet: - """Return the reference set with validated review decisions.""" - - -_ALIASES = { - "scope": ("page", "page scope", "scope", "page name"), - "url": ("url", "page url", "web address"), - "problem": ( - "problem", - "issue", - "issue title", - "defect", - "description", - ), - "location": ("location", "element", "element name", "where"), - "wcag": ( - "wcag", - "wcag sc", - "wcag criteria", - "success criterion", - "criterion", - ), - "recommendation": ( - "recommendation", - "recommendation for fix", - "remediation", - "fix", - ), -} -ELIGIBILITY_VALUES = ( - "llm_eligible", - "programmatic", - "unavailable_evidence", - "ambiguous", +SHEET = "Defect report" +HEADERS = ( + "Sr. #", "element name", "Browser Combination", "Page name", + "Issue Title", "Steps to Reproduce", "actual result", "expected result", + "Recommendation for Fix", "WCAG Sc", "Type of Change", "comment", ) -DECISION_VALUES = ("accepted", "rejected", "needs_review") -ELIGIBILITY_COLUMNS = ( - "reference_id", - "source_sheet", - "source_row", - "page_scope", - "canonical_url", - "problem", - "location", - "wcag_evidence", - "recommendation", - "raw_evidence", - "classification", - "decision", - "rationale", - "reviewer", - "confidence", - "timestamp", - "required_subdefects", - "notes", +ELIGIBILITY_VALUES = ( + "llm_eligible", "programmatic", "unavailable_evidence", "ambiguous", ) -def _text(value: object) -> str: - """Retain workbook cell content as text without semantic rewriting.""" - if value is None: - return "" - return str(value) - - -def _canonical_url(value: str) -> str: - """Normalize URL identity while discarding query and fragment drift.""" - parts = urlsplit(value.strip()) - if not parts.scheme or not parts.netloc: - return value.strip() - path = parts.path or "/" - return urlunsplit( - (parts.scheme.lower(), parts.netloc.lower(), path, "", "") - ) - - -def _field(headers: dict[str, str], raw: dict[str, str], name: str) -> str: - """Read one canonical field through the supported workbook aliases.""" - for alias in _ALIASES[name]: - header = headers.get(alias) - if header is not None: - return raw.get(header, "") - return "" - - -def _wcag_values(value: str) -> tuple[str, ...]: - """Extract ordered WCAG success-criterion identifiers.""" - return tuple(dict.fromkeys(re.findall(r"\b\d+\.\d+\.\d+\b", value))) - - -def import_homepage_references( - workbook_path: Path, - *, - expected_sha256: str, - homepage_url: str, - version: str, -) -> ReferenceSet: - """Import every Home and Global row, retaining source text verbatim.""" - if not workbook_path.is_file(): - raise ReferenceSetError( - f"Private workbook not found at {workbook_path}. Obtain it from " - "an " - "authorized project source or pass an explicit local path." - ) - actual_sha256 = hashlib.sha256(workbook_path.read_bytes()).hexdigest() - if actual_sha256.lower() != expected_sha256.lower(): - raise ReferenceSetError( - "Private workbook checksum drift detected: expected " - f"{expected_sha256}, found {actual_sha256}. Review and record " - "the new " - "authorized workbook before importing it." - ) - +def import_homepage_references(workbook_path: Path, homepage_url: str) -> ReferenceSet: + """Preserve every applicable source row and its original cell text.""" + checksum = hashlib.sha256(workbook_path.read_bytes()).hexdigest() + book = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) try: - workbook = openpyxl.load_workbook( - workbook_path, read_only=True, data_only=True - ) - except (OSError, ValueError, TypeError) as error: - raise ReferenceSetError( - f"Could not read private workbook: {error}" - ) from error - - canonical_homepage = _canonical_url(homepage_url) - imported: list[ReferenceDefect] = [] - for sheet in workbook.worksheets: + sheet = book[SHEET] rows = sheet.iter_rows(values_only=True) - first = next(rows, None) - if first is None: - continue - header_names = [_text(value).strip() for value in first] - normalized_headers = { - header.casefold(): header for header in header_names if header - } - if not any(alias in normalized_headers for alias in _ALIASES["scope"]): - continue - for row_number, values in enumerate(rows, start=2): + headers = tuple(str(value or "") for value in next(rows)) + if headers[:len(HEADERS)] != HEADERS: + raise ValueError("Reference workbook has unexpected Defect report columns") + references = [] + for number, values in enumerate(rows, start=2): raw = { - header: _text(value) - for header, value in zip(header_names, values, strict=False) - if header - } - scope_text = ( - _field(normalized_headers, raw, "scope").strip().casefold() - ) - row_url = _canonical_url(_field(normalized_headers, raw, "url")) - is_global = scope_text == "global" or row_url.casefold() in { - "global", - "all pages", - "sitewide", - "site-wide", + header: "" if value is None else str(value) + for header, value in zip(headers, values, strict=False) if header } - is_home = scope_text in {"home", "homepage", "home page"} - if not is_home and not is_global and row_url != canonical_homepage: + scope = raw["Page name"].strip().casefold() + if scope not in {"home", "global"}: continue - page_scope: Literal["home", "global"] = ( - "global" if is_global else "home" - ) - identity = f"{actual_sha256}:{sheet.title}:{row_number}".encode() - imported.append( - ReferenceDefect( - reference_id="ref-" - + hashlib.sha256(identity).hexdigest()[:16], - workbook_filename=workbook_path.name, - workbook_sha256=actual_sha256, - source_sheet=sheet.title, - source_row=row_number, - raw_evidence=raw, - canonical_url=canonical_homepage, - page_scope=page_scope, - problem=_field(normalized_headers, raw, "problem"), - location=_field(normalized_headers, raw, "location"), - wcag_evidence=_wcag_values( - _field(normalized_headers, raw, "wcag") - ), - recommendation=_field( - normalized_headers, raw, "recommendation" - ), - ) - ) - workbook.close() - if not imported: - raise ReferenceSetError( - "The workbook contained no Home or applicable Global rows. " - "Check the " - "homepage URL and workbook column names." - ) - return ReferenceSet( - version=version, - workbook_filename=workbook_path.name, - workbook_sha256=actual_sha256, - homepage_url=canonical_homepage, - references=tuple(imported), - ) - - -def export_eligibility_workbook( - reference_set: ReferenceSet, - output_path: Path, -) -> None: - """Create a private, multiline-friendly eligibility review workbook.""" - workbook = openpyxl.Workbook() - sheet = workbook.active - sheet.title = "Eligibility" - sheet.append(ELIGIBILITY_COLUMNS) - for reference in reference_set.references: - sheet.append( - ( - reference.reference_id, - reference.source_sheet, - reference.source_row, - reference.page_scope, - reference.canonical_url, - reference.problem, - reference.location, - ", ".join(reference.wcag_evidence), - reference.recommendation, - json.dumps( - reference.raw_evidence, ensure_ascii=False, sort_keys=True - ), - reference.eligibility or "", - reference.eligibility_state or "needs_review", - ( - reference.eligibility_review.rationale - if reference.eligibility_review - else "" - ), - ( - reference.eligibility_review.reviewer - if reference.eligibility_review - else "" - ), - ( - reference.eligibility_review.confidence - if reference.eligibility_review - else "" - ), - ( - reference.eligibility_review.timestamp - if reference.eligibility_review - else "" - ), - "\n".join(reference.required_subdefects), - "", - ) - ) - sheet.freeze_panes = "A2" - sheet.auto_filter.ref = sheet.dimensions - sheet.column_dimensions["F"].width = 45 - sheet.column_dimensions["G"].width = 32 - sheet.column_dimensions["J"].width = 60 - sheet.column_dimensions["M"].width = 45 - for row in sheet.iter_rows(min_row=2): - for cell in row: - cell.alignment = openpyxl.styles.Alignment( - wrap_text=True, vertical="top" - ) - classification_validation = DataValidation( - type="list", - formula1='"' + ",".join(ELIGIBILITY_VALUES) + '"', - ) - decision_validation = DataValidation( - type="list", - formula1='"' + ",".join(DECISION_VALUES) + '"', - ) - sheet.add_data_validation(classification_validation) - sheet.add_data_validation(decision_validation) - classification_validation.add(f"K2:K{max(2, sheet.max_row)}") - decision_validation.add(f"L2:L{max(2, sheet.max_row)}") - output_path.parent.mkdir(parents=True, exist_ok=True) - workbook.save(output_path) - - -class HumanEligibilityReviewer: - """Load eligibility decisions from the private review workbook.""" - - def __init__(self, review_path: Path): - """Select the private Eligibility workbook to read.""" - self.review_path = review_path - - def review(self, reference_set: ReferenceSet) -> ReferenceSet: - """Validate and apply exactly one decision for every source row.""" - try: - workbook = openpyxl.load_workbook( - self.review_path, - read_only=True, - data_only=True, - ) - sheet = workbook["Eligibility"] - except (OSError, KeyError, ValueError) as error: - raise ReferenceSetError( - f"Could not read Eligibility review workbook: {error}" - ) from error - rows = sheet.iter_rows(values_only=True) - raw_headers = next(rows, None) - headers = [_text(value).strip() for value in raw_headers or ()] - missing_columns = set(ELIGIBILITY_COLUMNS) - set(headers) - if missing_columns: - raise ReferenceSetError( - "Eligibility review is missing columns: " - + ", ".join(sorted(missing_columns)) - ) - decisions: dict[str, dict[str, str]] = {} - for row_number, values in enumerate(rows, start=2): - record = { - header: _text(value).strip() - for header, value in zip(headers, values, strict=False) - } - reference_id = record["reference_id"] - if not reference_id: - continue - if reference_id in decisions: - raise ReferenceSetError( - f"Eligibility review repeats reference {reference_id}." - ) - classification = record["classification"] - state = record["decision"] - if classification not in ELIGIBILITY_VALUES: - raise ReferenceSetError( - f"Eligibility row {row_number} has invalid classification " - f"{classification!r}." - ) - if state not in DECISION_VALUES: - raise ReferenceSetError( - f"Eligibility row {row_number} has invalid decision " - f"{state!r}." - ) - for field in ("rationale", "reviewer", "timestamp"): - if not record[field]: - raise ReferenceSetError( - f"Eligibility row {row_number} requires {field}." - ) - try: - confidence = float(record["confidence"]) - except ValueError as error: - raise ReferenceSetError( - f"Eligibility row {row_number} requires numeric " - "confidence." - ) from error - if not 0 <= confidence <= 1: - raise ReferenceSetError( - f"Eligibility row {row_number} confidence must be 0 " - "through 1." - ) - record["confidence"] = str(confidence) - decisions[reference_id] = record - workbook.close() - - expected_ids = {item.reference_id for item in reference_set.references} - if set(decisions) != expected_ids: - missing = sorted(expected_ids - set(decisions)) - unknown = sorted(set(decisions) - expected_ids) - details = [] - if missing: - details.append("missing " + ", ".join(missing)) - if unknown: - details.append("unknown " + ", ".join(unknown)) - raise ReferenceSetError( - "Eligibility review must decide every reference exactly once: " - + "; ".join(details) - ) - - reviewed = [] - for reference in reference_set.references: - decision = decisions[reference.reference_id] - subdefects = tuple( - value.strip() - for value in decision["required_subdefects"].splitlines() - if value.strip() - ) - reviewed.append( - replace( - reference, - eligibility=decision["classification"], # type: ignore[arg-type] - eligibility_state=decision["decision"], # type: ignore[arg-type] - eligibility_review=ReviewProvenance( - reviewer=decision["reviewer"], - rationale=decision["rationale"], - confidence=float(decision["confidence"]), - timestamp=decision["timestamp"], - ), - required_subdefects=subdefects, - ) - ) - return replace(reference_set, references=tuple(reviewed)) + if not raw["Issue Title"].strip(): + raise ValueError(f"Defect report row {number} has no Issue Title") + identity = f"{checksum}:{SHEET}:{number}".encode() + references.append(ReferenceDefect( + reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], + source_sheet=SHEET, source_row=number, page_scope=scope, + source_element=raw["element name"], problem=raw["Issue Title"], + raw_evidence=raw, + wcag_evidence=tuple(dict.fromkeys( + re.findall(r"\b\d+\.\d+\.\d+\b", raw["WCAG Sc"]) + )), + )) + finally: + book.close() + if not references: + raise ValueError("Reference workbook has no Home or Global defects") + return ReferenceSet(workbook_path.name, checksum, homepage_url, tuple(references)) diff --git a/vision_aid/evaluation/reporting.py b/vision_aid/evaluation/reporting.py deleted file mode 100644 index ce45cd6..0000000 --- a/vision_aid/evaluation/reporting.py +++ /dev/null @@ -1,174 +0,0 @@ -"""JSON, CSV, and Markdown projections of one evaluation score.""" - -from __future__ import annotations - -import csv -import json -from dataclasses import asdict, dataclass -from decimal import Decimal -from pathlib import Path - -from vision_aid.evaluation.scoring import EvaluationScore - - -@dataclass(frozen=True) -class ReportPaths: - """Files written for one score object.""" - - json: Path - csv: Path - markdown: Path - - -def _percentage(metric: dict[str, int | str]) -> str: - """Format one metric rate for the human report.""" - return f"{Decimal(str(metric['rate'])) * 100:.2f}%" - - -def _markdown(value: object) -> str: - """Keep multiline evidence inside one safe Markdown table cell.""" - return str(value).replace("|", "\\|").replace("\n", "
") - - -def write_reports( - score: EvaluationScore, output_directory: Path -) -> ReportPaths: - """Write three consistent views without recalculating any score.""" - output_directory.mkdir(parents=True, exist_ok=True) - base = output_directory / score.run_id - json_path = base.with_suffix(".json") - csv_path = base.with_suffix(".csv") - markdown_path = base.with_suffix(".md") - json_path.write_text( - json.dumps( - score.to_dict(), indent=2, sort_keys=True, ensure_ascii=False - ) - + "\n", - encoding="utf-8", - ) - common = { - "run_id": score.run_id, - "model": score.model, - "complete": score.complete, - "rankable": score.rankable, - "workbook_recall_caught": score.workbook_row_recall["caught"], - "workbook_recall_denominator": score.workbook_row_recall[ - "denominator" - ], - "workbook_recall_rate": score.workbook_row_recall["rate"], - "programmatic_coverage_caught": score.programmatic_coverage["caught"], - "programmatic_coverage_denominator": score.programmatic_coverage[ - "denominator" - ], - "programmatic_coverage_rate": score.programmatic_coverage["rate"], - "combined_caught": score.combined_workbook_coverage["caught"], - "combined_denominator": score.combined_workbook_coverage[ - "denominator" - ], - "combined_rate": score.combined_workbook_coverage["rate"], - "audit_cost_usd": score.audit_cost_usd, - "evaluation_cost_usd": score.evaluation_cost_usd, - "total_experiment_cost_usd": score.total_experiment_cost_usd, - **score.usage, - "wall_time_seconds": score.wall_time_seconds, - "request_duration_sum_seconds": score.request_duration_sum_seconds, - "request_durations_seconds": json.dumps( - score.request_durations_seconds - ), - "parse_failure_count": len(score.parse_failures), - "unmatched_audit_finding_count": len( - score.unmatched_audit_finding_ids - ), - } - row_dicts = [{**common, **asdict(row)} for row in score.rows] - with csv_path.open("w", newline="", encoding="utf-8") as stream: - if row_dicts: - writer = csv.DictWriter(stream, fieldnames=list(row_dicts[0])) - writer.writeheader() - for data_row in row_dicts: - data_row["model_finding_ids"] = "; ".join( - data_row["model_finding_ids"] - ) - data_row["programmatic_finding_ids"] = "; ".join( - data_row["programmatic_finding_ids"] - ) - data_row["model_evidence"] = "; ".join( - data_row["model_evidence"] - ) - data_row["programmatic_evidence"] = "; ".join( - data_row["programmatic_evidence"] - ) - writer.writerow(data_row) - recall = score.workbook_row_recall - programmatic = score.programmatic_coverage - combined = score.combined_workbook_coverage - lines = [ - f"# Illustrative model evaluation: {score.model}", - "", - ( - "> This homepage result is illustrative and does not establish " - "broad " - "small-versus-large model equivalence." - ), - "", - ( - "- Workbook-row recall: " - f"{recall['caught']}/{recall['denominator']} " - f"({_percentage(recall)})" - ), - ( - f"- Programmatic coverage: {programmatic['caught']}/" - f"{programmatic['denominator']} ({_percentage(programmatic)})" - ), - ( - f"- Combined workbook coverage: {combined['caught']}/" - f"{combined['denominator']} ({_percentage(combined)})" - ), - f"- Estimated audit cost: ${score.audit_cost_usd}", - f"- Estimated evaluation cost: ${score.evaluation_cost_usd}", - ( - "- Estimated total experiment cost: " - f"${score.total_experiment_cost_usd}" - ), - f"- Input tokens: {score.usage['input_tokens']}", - f"- Cached input tokens: {score.usage['cached_input_tokens']}", - f"- Output tokens: {score.usage['output_tokens']}", - "- Reasoning tokens: " + ( - "unavailable (included in aggregate output)" - if score.usage['reasoning_tokens'] is None - else str(score.usage['reasoning_tokens']) - ), - ( - "- Cache-creation input tokens: " - f"{score.usage['cache_creation_input_tokens']}" - ), - f"- End-to-end wall time: {score.wall_time_seconds:g}s", - f"- Request duration sum: {score.request_duration_sum_seconds:g}s", - "- Per-request durations: " - + json.dumps(score.request_durations_seconds), - f"- Complete and rankable: {score.rankable}", - f"- Format failures: {len(score.parse_failures)}", - ( - "- Unmatched audit findings: " - f"{len(score.unmatched_audit_finding_ids)}" - ), - "", - ( - "| Source row | Problem | Location | Eligibility | Rationale | " - "Programmatic evidence | Model evidence | Disposition |" - ), - "|---|---|---|---|---|---|---|---|", - ] - for report_row in score.rows: - lines.append( - f"| {report_row.source_sheet}!{report_row.source_row} | " - f"{_markdown(report_row.problem)} | " - f"{_markdown(report_row.location)} | " - f"{report_row.eligibility} | " - f"{_markdown(report_row.eligibility_rationale)} | " - f"{_markdown('; '.join(report_row.programmatic_evidence))} | " - f"{_markdown('; '.join(report_row.model_evidence))} | " - f"{report_row.disposition} |" - ) - markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return ReportPaths(json=json_path, csv=csv_path, markdown=markdown_path) diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py new file mode 100644 index 0000000..e46c36a --- /dev/null +++ b/vision_aid/evaluation/review.py @@ -0,0 +1,156 @@ +"""One editable CSV for human decisions and one deterministic Markdown report.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from vision_aid.evaluation.references import ELIGIBILITY_VALUES, HEADERS + +DECISION_COLUMNS = ( + "classification", "classification_reason", "audit_finding_id", + "programmatic_finding_id", "match_reason", "review_notes", +) +REVIEW_COLUMNS = ( + "reference_id", "source_sheet", "source_row", "page_scope", + "source_element", *HEADERS, *DECISION_COLUMNS, +) + + +def create_review(path: Path, references: list[dict[str, Any]]) -> None: + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=REVIEW_COLUMNS) + writer.writeheader() + for item in references: + writer.writerow({ + "reference_id": item["reference_id"], + "source_sheet": item["source_sheet"], + "source_row": item["source_row"], + "page_scope": item["page_scope"], + "source_element": item["source_element"], + **item["raw_evidence"], + **{column: "" for column in DECISION_COLUMNS}, + }) + + +def _ids(value: str) -> tuple[str, ...]: + """Semicolons join multiple finding IDs in either finding column.""" + parts = tuple(part.strip() for part in value.split(";") if part.strip()) + if len(parts) != len(set(parts)): + raise ValueError("A finding ID is repeated within one review row") + return parts + + +def reviewed_rows( + path: Path, references: list[dict[str, Any]], + audit: list[dict[str, Any]], programmatic: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Validate completeness and finding kind before granting any row credit.""" + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + if not set(REVIEW_COLUMNS).issubset(reader.fieldnames or ()): + raise ValueError("Review CSV is missing required columns") + entered = list(reader) + expected = {item["reference_id"]: item for item in references} + if len(entered) != len(expected): + raise ValueError("Review CSV must contain each Reference defect exactly once") + by_kind = { + "audit": {item["finding_id"]: item for item in audit}, + "programmatic": {item["finding_id"]: item for item in programmatic}, + } + seen_rows: set[str] = set() + used: dict[str, set[str]] = {"audit": set(), "programmatic": set()} + result = [] + for row in entered: + reference_id = row["reference_id"] + if reference_id not in expected or reference_id in seen_rows: + raise ValueError(f"Unknown or repeated Reference defect: {reference_id}") + seen_rows.add(reference_id) + reference = expected[reference_id] + for field in ("source_sheet", "source_row", "source_element"): + if str(row[field]) != str(reference[field]): + raise ValueError(f"Reference identity changed for {reference_id}: {field}") + classification = row["classification"].strip() + if classification not in ELIGIBILITY_VALUES: + raise ValueError(f"Reference {reference_id} needs a valid classification") + if not row["classification_reason"].strip(): + raise ValueError(f"Reference {reference_id} needs a classification reason") + matched: dict[str, tuple[str, ...]] = {} + for kind, column in ( + ("audit", "audit_finding_id"), ("programmatic", "programmatic_finding_id") + ): + ids = _ids(row[column]) + unknown = set(ids) - set(by_kind[kind]) + if unknown: + raise ValueError(f"Unknown {kind} finding ID: {', '.join(sorted(unknown))}") + reused = set(ids) & used[kind] + if reused: + raise ValueError(f"Conflicting {kind} finding ID: {', '.join(sorted(reused))}") + used[kind].update(ids) + matched[kind] = ids + if any(matched.values()) and not row["match_reason"].strip(): + raise ValueError(f"Reference {reference_id} needs a match reason") + if classification in {"unavailable_evidence", "ambiguous"} and any(matched.values()): + raise ValueError(f"Excluded Reference {reference_id} cannot claim coverage") + result.append({"reference": reference, "decision": row, "matches": matched}) + return result + + +def _metric(caught: int, total: int) -> str: + if total == 0: + return "N/A (0 eligible rows)" + return f"{caught}/{total} ({caught / total:.1%})" + + +def _md(value: object) -> str: + return str(value).replace("\\", "\\\\").replace("|", "\\|").replace("\n", "
") + + +def write_report( + path: Path, manifest: dict[str, Any], rows: list[dict[str, Any]], + audit: list[dict[str, Any]], programmatic: list[dict[str, Any]], +) -> None: + if not manifest["complete"]: + raise ValueError("Incomplete Evaluation runs cannot produce a final report") + findings = {item["finding_id"]: item for item in audit + programmatic} + llm = [row for row in rows if row["decision"]["classification"] == "llm_eligible"] + checks = [row for row in rows if row["decision"]["classification"] == "programmatic"] + covered = [row for row in rows if any(row["matches"].values())] + unavailable = sum(row["decision"]["classification"] == "unavailable_evidence" for row in rows) + ambiguous = sum(row["decision"]["classification"] == "ambiguous" for row in rows) + lines = [ + f"# Evaluation run {manifest['run_id']}", "", + f"- Model: {manifest['configuration']['model']}", + f"- Workbook-row recall: {_metric(sum(bool(row['matches']['audit']) for row in llm), len(llm))}", + f"- Programmatic coverage: {_metric(sum(bool(row['matches']['programmatic']) for row in checks), len(checks))}", + f"- Combined workbook coverage: {_metric(len(covered), len(rows))}", + f"- Estimated run cost: ${manifest['estimated_cost_usd']}", + f"- Unavailable evidence: {unavailable}; ambiguous: {ambiguous}", + f"- Reported usage: {_md(json.dumps(manifest['usage'], sort_keys=True))}", + "", "This is full-row coverage of this homepage workbook only. A blank finding cell means missed.", + "The cost guardrail is checked between requests; a request can exceed the remaining amount.", + f"Malformed response prompts: {_md(', '.join(manifest['format_failures']) or 'none')}", + "", "## Reference defects", "", + ] + for row in rows: + reference = row["reference"] + decision = row["decision"] + matches = row["matches"] + lines.extend([ + f"### {reference['reference_id']} — {_md(reference['problem'])}", "", + f"- Source: {reference['source_sheet']} row {reference['source_row']} ({reference['page_scope']})", + f"- source_element: {_md(reference['source_element'])}", + f"- Original defect evidence: {_md(json.dumps(reference['raw_evidence'], ensure_ascii=False))}", + f"- Classification: {decision['classification']} — {_md(decision['classification_reason'])}", + f"- Audit findings: {_md('; '.join(matches['audit']) or 'none')}", + f"- Programmatic findings: {_md('; '.join(matches['programmatic']) or 'none')}", + f"- Match reason: {_md(decision['match_reason'] or 'none')}", + f"- Review notes: {_md(decision['review_notes'] or 'none')}", + ]) + for finding_id in (*matches["audit"], *matches["programmatic"]): + finding = findings[finding_id] + lines.append(f"- {finding_id}: {_md(finding['problem'])} @ {_md(finding['location'])}") + lines.append("") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index 61cb710..9c04140 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -1,70 +1,36 @@ -"""Canonical, JSON-compatible records shared by evaluation stages.""" +"""Evidence records for the fixed homepage evaluation.""" from __future__ import annotations -from dataclasses import asdict, dataclass, field +from dataclasses import asdict, dataclass from typing import Any, Literal -Eligibility = Literal[ - "llm_eligible", - "programmatic", - "unavailable_evidence", - "ambiguous", -] -DecisionState = Literal["accepted", "rejected", "needs_review"] - - -@dataclass(frozen=True) -class ReviewProvenance: - """Who made a review decision and why.""" - - reviewer: str - rationale: str - confidence: float - timestamp: str - @dataclass(frozen=True) class ReferenceDefect: - """One workbook source row retained as one benchmark scoring unit.""" - reference_id: str - workbook_filename: str - workbook_sha256: str source_sheet: str source_row: int - raw_evidence: dict[str, str] - canonical_url: str page_scope: Literal["home", "global"] + source_element: str problem: str - location: str + raw_evidence: dict[str, str] wcag_evidence: tuple[str, ...] - recommendation: str = "" - required_subdefects: tuple[str, ...] = () - eligibility: Eligibility | None = None - eligibility_state: DecisionState | None = None - eligibility_review: ReviewProvenance | None = None @dataclass(frozen=True) class ReferenceSet: - """Versioned collection of imported or reviewed reference defects.""" - - version: str workbook_filename: str workbook_sha256: str homepage_url: str references: tuple[ReferenceDefect, ...] def to_dict(self) -> dict[str, Any]: - """Return a stable JSON-compatible representation.""" return asdict(self) @dataclass(frozen=True) class CanonicalFinding: - """Lossless normalized output from one audit or programmatic source.""" - finding_id: str source: Literal["audit", "programmatic"] run_id: str @@ -79,53 +45,3 @@ class CanonicalFinding: wcag_evidence: tuple[str, ...] raw_source: Any parse_status: str - - -@dataclass(frozen=True) -class MatchDecision: - """Reviewed relationship between one finding and one reference defect.""" - - reference_id: str - finding_id: str - state: DecisionState - provenance: ReviewProvenance - required_subdefect: str | None = None - page_compatible: bool | None = None - failure_compatible: bool | None = None - location_compatible: bool | None = None - notes: str = "" - - -@dataclass(frozen=True) -class Usage: - """Token categories reported by providers across all attempts.""" - - input_tokens: int = 0 - cached_input_tokens: int = 0 - output_tokens: int = 0 - reasoning_tokens: int | None = 0 - cache_creation_input_tokens: int = 0 - - -@dataclass(frozen=True) -class RunMetadata: - """Operational facts used by deterministic scoring and comparison.""" - - run_id: str - model: str - complete: bool - comparable_identity: str - audit_cost_usd: str - evaluation_cost_usd: str = "0" - usage: Usage = field(default_factory=Usage) - wall_time_seconds: float = 0 - request_durations_seconds: tuple[float, ...] = () - workbook_sha256: str = "" - snapshot_sha256: str = "" - reference_set_version: str = "" - prompt_hashes_identity: str = "" - parser_identity: str = "" - eligibility_identity: str = "" - configuration_identity: str = "" - homepage_url: str = "" - prompt_names: tuple[str, ...] = () diff --git a/vision_aid/evaluation/scoring.py b/vision_aid/evaluation/scoring.py deleted file mode 100644 index 68c9005..0000000 --- a/vision_aid/evaluation/scoring.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Network-free deterministic workbook-row scoring and ranking.""" - -from __future__ import annotations - -from collections.abc import Sequence -from dataclasses import asdict, dataclass -from decimal import Decimal -from typing import Any - -from vision_aid.evaluation.matching import validate_match_decisions -from vision_aid.evaluation.schemas import ( - CanonicalFinding, - MatchDecision, - ReferenceDefect, - ReferenceSet, - RunMetadata, -) - - -def _metric(caught: int, denominator: int) -> dict[str, int | str]: - """Represent one exact count, denominator, and decimal rate.""" - rate = ( - Decimal(caught) / Decimal(denominator) if denominator else Decimal(0) - ) - return {"caught": caught, "denominator": denominator, "rate": str(rate)} - - -@dataclass(frozen=True) -class RowScore: - """Explain the final disposition for one workbook source row.""" - - reference_id: str - source_sheet: str - source_row: int - page_scope: str - eligibility: str - eligibility_rationale: str - problem: str - location: str - programmatic_caught: bool - model_caught: bool - model_finding_ids: tuple[str, ...] - programmatic_finding_ids: tuple[str, ...] - model_evidence: tuple[str, ...] - programmatic_evidence: tuple[str, ...] - disposition: str - - -@dataclass(frozen=True) -class EvaluationScore: - """Complete deterministic source object for every report projection.""" - - run_id: str - model: str - complete: bool - rankable: bool - comparable_identity: str - compatibility: dict[str, str] - reference_set_version: str - workbook_sha256: str - workbook_row_recall: dict[str, int | str] - programmatic_coverage: dict[str, int | str] - combined_workbook_coverage: dict[str, int | str] - audit_cost_usd: str - evaluation_cost_usd: str - total_experiment_cost_usd: str - usage: dict[str, int | None] - wall_time_seconds: float - request_durations_seconds: tuple[float, ...] - request_duration_sum_seconds: float - unmatched_audit_finding_ids: tuple[str, ...] - parse_failures: tuple[str, ...] - rows: tuple[RowScore, ...] - - def to_dict(self) -> dict[str, Any]: - """Return the stable JSON-compatible score representation.""" - return asdict(self) - - -def _caught_references( - references: Sequence[ReferenceDefect], - decisions: Sequence[MatchDecision], -) -> tuple[set[str], dict[str, tuple[str, ...]]]: - """Calculate binary row credit and its accepted finding evidence.""" - accepted = [item for item in decisions if item.state == "accepted"] - by_reference: dict[str, list[MatchDecision]] = {} - for decision in accepted: - by_reference.setdefault(decision.reference_id, []).append(decision) - caught: set[str] = set() - evidence: dict[str, tuple[str, ...]] = {} - for reference in references: - matches = by_reference.get(reference.reference_id, []) - if reference.required_subdefects: - matched_parts = {item.required_subdefect for item in matches} - is_caught = set(reference.required_subdefects) <= matched_parts - else: - is_caught = bool(matches) - if is_caught: - caught.add(reference.reference_id) - evidence[reference.reference_id] = tuple( - sorted(item.finding_id for item in matches) - ) - return caught, evidence - - -def score_evaluation( - reference_set: ReferenceSet, - audit_findings: Sequence[CanonicalFinding], - programmatic_findings: Sequence[CanonicalFinding], - audit_matches: Sequence[MatchDecision], - programmatic_matches: Sequence[MatchDecision], - run: RunMetadata, - *, - parse_failures: Sequence[str] = (), -) -> EvaluationScore: - """Calculate binary workbook-row credit from approved normalized inputs.""" - invalid_audit_findings = [ - item.finding_id - for item in audit_findings - if item.source != "audit" - or item.run_id != run.run_id - or item.model != run.model - or (run.homepage_url and item.page_url != run.homepage_url) - or (run.prompt_names and item.prompt not in run.prompt_names) - ] - if invalid_audit_findings: - raise ValueError( - "Audit findings do not belong to the verified run: " - + ", ".join(invalid_audit_findings) - ) - invalid_programmatic_findings = [ - item.finding_id - for item in programmatic_findings - if item.source != "programmatic" - or item.run_id != run.run_id - or item.model is not None - or (run.homepage_url and item.page_url != run.homepage_url) - ] - if invalid_programmatic_findings: - raise ValueError( - "Programmatic findings do not belong to the verified run: " - + ", ".join(invalid_programmatic_findings) - ) - unapproved = [ - item.reference_id - for item in reference_set.references - if item.eligibility_state != "accepted" or item.eligibility is None - ] - if unapproved: - raise ValueError( - "Scoring requires accepted eligibility decisions for every " - "reference: " + ", ".join(unapproved) - ) - audit_matches = validate_match_decisions( - reference_set.references, - audit_findings, - audit_matches, - ) - programmatic_matches = validate_match_decisions( - reference_set.references, - programmatic_findings, - programmatic_matches, - ) - model_caught, model_evidence = _caught_references( - reference_set.references, audit_matches - ) - programmatic_caught, programmatic_evidence = _caught_references( - reference_set.references, programmatic_matches - ) - audit_finding_by_id = {item.finding_id: item for item in audit_findings} - programmatic_finding_by_id = { - item.finding_id: item for item in programmatic_findings - } - llm_references = { - item.reference_id - for item in reference_set.references - if item.eligibility == "llm_eligible" - } - programmatic_references = { - item.reference_id - for item in reference_set.references - if item.eligibility == "programmatic" - } - combined = model_caught | programmatic_caught - rows = [] - for reference in reference_set.references: - model_hit = reference.reference_id in model_caught - programmatic_hit = reference.reference_id in programmatic_caught - if model_hit and programmatic_hit: - disposition = "caught_by_both" - elif model_hit: - disposition = "caught_by_model" - elif programmatic_hit: - disposition = "caught_programmatically" - elif reference.eligibility == "programmatic": - disposition = "programmatic_gap" - elif reference.eligibility == "unavailable_evidence": - disposition = "unavailable_evidence" - elif reference.eligibility == "ambiguous": - disposition = "ambiguous" - else: - disposition = "missed_by_model" - rows.append( - RowScore( - reference_id=reference.reference_id, - source_sheet=reference.source_sheet, - source_row=reference.source_row, - page_scope=reference.page_scope, - eligibility=reference.eligibility or "", - eligibility_rationale=( - reference.eligibility_review.rationale - if reference.eligibility_review - else "" - ), - problem=reference.problem, - location=reference.location, - programmatic_caught=programmatic_hit, - model_caught=model_hit, - model_finding_ids=model_evidence[reference.reference_id], - programmatic_finding_ids=programmatic_evidence[ - reference.reference_id - ], - model_evidence=tuple( - f"{audit_finding_by_id[finding_id].problem} @ " - f"{audit_finding_by_id[finding_id].location}" - for finding_id in model_evidence[reference.reference_id] - ), - programmatic_evidence=tuple( - f"{programmatic_finding_by_id[finding_id].problem} @ " - f"{programmatic_finding_by_id[finding_id].location}" - for finding_id in programmatic_evidence[ - reference.reference_id - ] - ), - disposition=disposition, - ) - ) - accepted_audit_ids = { - item.finding_id for item in audit_matches if item.state == "accepted" - } - total_cost = Decimal(run.audit_cost_usd) + Decimal(run.evaluation_cost_usd) - if ( - run.reference_set_version - and run.reference_set_version != reference_set.version - ): - raise ValueError( - "Run metadata reference-set version does not match the scored " - "input." - ) - if ( - run.workbook_sha256 - and run.workbook_sha256 != reference_set.workbook_sha256 - ): - raise ValueError( - "Run metadata workbook checksum does not match the scored input." - ) - return EvaluationScore( - run_id=run.run_id, - model=run.model, - complete=run.complete, - rankable=run.complete, - comparable_identity=run.comparable_identity, - compatibility={ - "benchmark": run.comparable_identity, - "workbook": reference_set.workbook_sha256, - "snapshot": run.snapshot_sha256, - "reference_set": reference_set.version, - "prompts": run.prompt_hashes_identity, - "parser": run.parser_identity, - "eligibility": run.eligibility_identity, - "configuration": run.configuration_identity, - }, - reference_set_version=reference_set.version, - workbook_sha256=reference_set.workbook_sha256, - workbook_row_recall=_metric( - len(model_caught & llm_references), len(llm_references) - ), - programmatic_coverage=_metric( - len(programmatic_caught & programmatic_references), - len(programmatic_references), - ), - combined_workbook_coverage=_metric( - len(combined), len(reference_set.references) - ), - audit_cost_usd=run.audit_cost_usd, - evaluation_cost_usd=run.evaluation_cost_usd, - total_experiment_cost_usd=str(total_cost), - usage=asdict(run.usage), - wall_time_seconds=run.wall_time_seconds, - request_durations_seconds=run.request_durations_seconds, - request_duration_sum_seconds=sum(run.request_durations_seconds), - unmatched_audit_finding_ids=tuple( - sorted( - item.finding_id - for item in audit_findings - if item.finding_id not in accepted_audit_ids - ) - ), - parse_failures=tuple(sorted(parse_failures)), - rows=tuple(rows), - ) - - -def rank_scores( - scores: Sequence[EvaluationScore], -) -> tuple[EvaluationScore, ...]: - """Rank complete runs by recall, then unrounded audit cost only.""" - if any(not item.rankable for item in scores): - raise ValueError( - "An incomplete run is unranked and cannot be compared." - ) - identities = {tuple(sorted(item.compatibility.items())) for item in scores} - if len(identities) > 1: - raise ValueError( - "Runs use incompatible benchmark versions; cross-version analysis " - "must " - "be explicitly requested and labeled." - ) - return tuple( - sorted( - scores, - key=lambda item: ( - -Decimal(str(item.workbook_row_recall["rate"])), - Decimal(item.audit_cost_usd), - ), - ) - ) diff --git a/vision_aid/evaluation/serialization.py b/vision_aid/evaluation/serialization.py deleted file mode 100644 index 3f14ad6..0000000 --- a/vision_aid/evaluation/serialization.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Validated JSON serialization at private evaluation workflow boundaries.""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from vision_aid.evaluation.schemas import ( - CanonicalFinding, - MatchDecision, - ReferenceDefect, - ReferenceSet, - ReviewProvenance, - RunMetadata, - Usage, -) - - -def write_json(path: Path, value: object) -> None: - """Write stable, human-readable JSON to a private artifact path.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - - -def json_digest(value: object) -> str: - """Hash a canonical JSON representation for content identity checks.""" - encoded = json.dumps( - value, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ).encode() - return hashlib.sha256(encoded).hexdigest() - - -def reference_set_identity(reference_set: ReferenceSet) -> str: - """Hash all approved reference and eligibility decision content.""" - return json_digest(reference_set.to_dict()) - - -def save_reference_set(path: Path, reference_set: ReferenceSet) -> None: - """Persist a canonical reference set.""" - write_json(path, reference_set.to_dict()) - - -def _provenance(value: dict[str, Any] | None) -> ReviewProvenance | None: - """Construct optional reviewer provenance from JSON data.""" - return ReviewProvenance(**value) if value else None - - -def load_reference_set(path: Path) -> ReferenceSet: - """Load a canonical reference set from a private artifact.""" - data = json.loads(path.read_text(encoding="utf-8")) - references = [] - for value in data["references"]: - value = dict(value) - value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) - value["required_subdefects"] = tuple(value.get("required_subdefects", ())) - value["eligibility_review"] = _provenance(value.get("eligibility_review")) - references.append(ReferenceDefect(**value)) - return ReferenceSet( - version=data["version"], - workbook_filename=data["workbook_filename"], - workbook_sha256=data["workbook_sha256"], - homepage_url=data["homepage_url"], - references=tuple(references), - ) - - -def save_findings(path: Path, findings: tuple[CanonicalFinding, ...]) -> None: - """Persist canonical findings without changing their raw evidence.""" - write_json(path, [asdict(item) for item in findings]) - - -def load_findings(path: Path) -> tuple[CanonicalFinding, ...]: - """Load canonical audit or programmatic findings.""" - values = json.loads(path.read_text(encoding="utf-8")) - findings = [] - for value in values: - value["wcag_evidence"] = tuple(value.get("wcag_evidence", ())) - findings.append(CanonicalFinding(**value)) - return tuple(findings) - - -def save_match_decisions(path: Path, decisions: tuple[MatchDecision, ...]) -> None: - """Persist reviewed match decisions.""" - write_json(path, [asdict(item) for item in decisions]) - - -def load_match_decisions(path: Path) -> tuple[MatchDecision, ...]: - """Load reviewed match decisions and their provenance.""" - values = json.loads(path.read_text(encoding="utf-8")) - decisions = [] - for value in values: - value["provenance"] = ReviewProvenance(**value["provenance"]) - decisions.append(MatchDecision(**value)) - return tuple(decisions) - - -def load_run_metadata(value: dict[str, Any]) -> RunMetadata: - """Load synthetic run metadata from an in-memory test bundle.""" - value = dict(value) - value["usage"] = Usage(**value.get("usage", {})) - value["request_durations_seconds"] = tuple( - value.get("request_durations_seconds", ()) - ) - return RunMetadata(**value) - - -def load_verified_run_metadata( - path: Path, - reference_set: ReferenceSet, -) -> RunMetadata: - """Derive score metadata from a manifest and verify every identity.""" - manifest = json.loads(path.read_text(encoding="utf-8")) - identity_fields = { - key: manifest[key] - for key in ( - "workbook_sha256", - "snapshot_sha256", - "reference_set_version", - "eligibility_identity", - "configuration", - "filters", - "prompt_hashes", - "parser_hash", - "pricing_identity", - ) - } - benchmark_identity = json_digest(identity_fields) - if manifest.get("benchmark_identity") != benchmark_identity: - raise ValueError( - "Run manifest benchmark identities are inconsistent or changed." - ) - if manifest["workbook_sha256"] != reference_set.workbook_sha256: - raise ValueError("Run manifest workbook checksum does not match references.") - if manifest["reference_set_version"] != reference_set.version: - raise ValueError( - "Run manifest reference-set version does not match references." - ) - expected_eligibility = reference_set_identity(reference_set) - if manifest["eligibility_identity"] != expected_eligibility: - raise ValueError("Run manifest eligibility decisions do not match references.") - usage_value = manifest.get("usage", {}) - reasoning_tokens = usage_value.get("reasoning_tokens", 0) - usage = Usage( - reasoning_tokens=None if reasoning_tokens is None else int(reasoning_tokens), - **{ - key: int(usage_value.get(key, 0)) - for key in ( - "input_tokens", - "cached_input_tokens", - "output_tokens", - "cache_creation_input_tokens", - ) - }, - ) - return RunMetadata( - run_id=str(manifest["run_id"]), - model=str(manifest["configuration"]["model"]), - complete=bool(manifest.get("complete", False)), - comparable_identity=benchmark_identity, - audit_cost_usd=str(manifest.get("estimated_audit_cost_usd", "0")), - evaluation_cost_usd=str(manifest.get("evaluation_cost_usd", "0")), - usage=usage, - wall_time_seconds=float(manifest.get("wall_time_seconds", 0)), - request_durations_seconds=tuple( - float(value) for value in manifest.get("request_durations_seconds", ()) - ), - workbook_sha256=str(manifest["workbook_sha256"]), - snapshot_sha256=str(manifest["snapshot_sha256"]), - reference_set_version=str(manifest["reference_set_version"]), - prompt_hashes_identity=json_digest(manifest["prompt_hashes"]), - parser_identity=str(manifest["parser_hash"]), - eligibility_identity=str(manifest["eligibility_identity"]), - configuration_identity=json_digest( - { - "configuration": manifest["configuration"], - "filters": manifest["filters"], - } - ), - homepage_url=str(manifest["homepage_url"]), - prompt_names=tuple(item["name"] for item in manifest["prompts"]), - ) diff --git a/vision_aid/evaluation/snapshot.py b/vision_aid/evaluation/snapshot.py deleted file mode 100644 index 3e1686d..0000000 --- a/vision_aid/evaluation/snapshot.py +++ /dev/null @@ -1,307 +0,0 @@ -"""Immutable benchmark snapshots for model evaluation.""" - -import hashlib -import json -from dataclasses import asdict, dataclass -from datetime import UTC, datetime -from pathlib import Path -from tempfile import mkdtemp -from typing import cast -from urllib.parse import urlparse - -from vision_aid.evaluation.workspace import PrivateInputError, PrivateWorkspace - -SNAPSHOT_DIRECTORY_NAME = "pristine-homepage" -SNAPSHOT_HTML_FILENAME = "source.html" -SNAPSHOT_METADATA_FILENAME = "metadata.json" -DAT_VISION_AID_FIXTURE = "dat_visionaid_home.html" -HTTP_TIMEOUT = (5, 30) -PIPELINE_USER_AGENT = "Mozilla/5.0" - - -@dataclass(frozen=True) -class SnapshotHTTPMetadata: - """HTTP request and response details that identify captured content.""" - - content_length: str | None - content_type: str | None - etag: str | None - final_url: str - last_modified: str | None - request_user_agent: str - status_code: int - - -@dataclass(frozen=True) -class SnapshotMetadata: - """Required provenance for one immutable benchmark snapshot.""" - - byte_length: int - html_file: str - http: SnapshotHTTPMetadata - retrieved_at: str - sha256: str - source_url: str - - def to_json(self) -> object: - """Return the JSON-compatible representation stored beside the HTML.""" - return asdict(self) - - @classmethod - def from_json(cls, value: object) -> "SnapshotMetadata": - """Validate and construct snapshot provenance loaded from JSON.""" - if not isinstance(value, dict): - raise ValueError("snapshot metadata must be a JSON object") - http_value = value.get("http") - if not isinstance(http_value, dict): - raise ValueError("snapshot HTTP metadata must be a JSON object") - - required_strings = ( - "html_file", - "retrieved_at", - "sha256", - "source_url", - ) - if any(not isinstance(value.get(name), str) for name in required_strings): - raise ValueError("snapshot metadata has a missing string field") - required_http_strings = ( - "final_url", - "request_user_agent", - ) - if any( - not isinstance(http_value.get(name), str) - for name in required_http_strings - ): - raise ValueError("snapshot HTTP metadata has a missing string field") - optional_http_strings = ( - "content_length", - "content_type", - "etag", - "last_modified", - ) - if any( - http_value.get(name) is not None - and not isinstance(http_value.get(name), str) - for name in optional_http_strings - ): - raise ValueError("snapshot HTTP metadata has an invalid header field") - byte_length = value.get("byte_length") - status_code = http_value.get("status_code") - if not isinstance(byte_length, int) or isinstance(byte_length, bool): - raise ValueError("snapshot byte length must be an integer") - if not isinstance(status_code, int) or isinstance(status_code, bool): - raise ValueError("snapshot HTTP status must be an integer") - - return cls( - byte_length=byte_length, - html_file=cast(str, value["html_file"]), - http=SnapshotHTTPMetadata( - content_length=cast(str | None, http_value.get("content_length")), - content_type=cast(str | None, http_value.get("content_type")), - etag=cast(str | None, http_value.get("etag")), - final_url=cast(str, http_value["final_url"]), - last_modified=cast(str | None, http_value.get("last_modified")), - request_user_agent=cast(str, http_value["request_user_agent"]), - status_code=status_code, - ), - retrieved_at=cast(str, value["retrieved_at"]), - sha256=cast(str, value["sha256"]), - source_url=cast(str, value["source_url"]), - ) - - -@dataclass(frozen=True) -class BenchmarkSnapshot: - """A verified raw HTML benchmark and its provenance record.""" - - html_path: Path - metadata_path: Path - metadata: SnapshotMetadata - reused: bool - - -def prepare_benchmark_snapshot( - workspace: PrivateWorkspace, - source_url: str, -) -> BenchmarkSnapshot: - """Capture a benchmark once or verify an existing immutable capture.""" - _require_http_source(source_url) - snapshot_directory = ( - workspace.root / "snapshots" / SNAPSHOT_DIRECTORY_NAME - ) - html_path = snapshot_directory / SNAPSHOT_HTML_FILENAME - metadata_path = snapshot_directory / SNAPSHOT_METADATA_FILENAME - - if snapshot_directory.exists(): - return _load_snapshot( - html_path, - metadata_path, - source_url, - ) - - import requests - - try: - response = requests.get( - source_url, - headers={"User-Agent": PIPELINE_USER_AGENT}, - timeout=HTTP_TIMEOUT, - ) - except requests.RequestException as error: - raise PrivateInputError( - f"Could not retrieve Pristine benchmark source {source_url}: {error}" - ) from error - - if response.status_code != 200: - raise PrivateInputError( - f"Pristine benchmark source returned HTTP {response.status_code}; " - "no snapshot was saved." - ) - content_type = response.headers.get("Content-Type") - if content_type is not None and not content_type.lower().startswith("text/html"): - raise PrivateInputError( - f"Pristine benchmark source returned {content_type!r}, not raw HTML; " - "no snapshot was saved." - ) - body = response.content - if not body: - raise PrivateInputError( - "Pristine benchmark source returned an empty body; no snapshot was saved." - ) - try: - body.decode("utf-8") - except UnicodeDecodeError as error: - raise PrivateInputError( - "Pristine benchmark HTML is not valid UTF-8 and cannot be consumed " - "by the existing audit pipeline; no snapshot was saved." - ) from error - - metadata = SnapshotMetadata( - byte_length=len(body), - html_file=SNAPSHOT_HTML_FILENAME, - http=SnapshotHTTPMetadata( - content_length=response.headers.get("Content-Length"), - content_type=content_type, - etag=response.headers.get("ETag"), - final_url=response.url, - last_modified=response.headers.get("Last-Modified"), - request_user_agent=PIPELINE_USER_AGENT, - status_code=response.status_code, - ), - retrieved_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"), - sha256=hashlib.sha256(body).hexdigest(), - source_url=source_url, - ) - _save_new_snapshot(snapshot_directory, body, metadata) - return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=False) - - -def _require_http_source(source_url: str) -> None: - """Reject local substitutes and malformed benchmark source URLs.""" - parsed = urlparse(source_url) - if parsed.scheme in {"http", "https"} and parsed.netloc: - return - if Path(source_url).name == DAT_VISION_AID_FIXTURE: - raise PrivateInputError( - "The DAT Vision Aid fixture is not a Pristine benchmark input. " - "Capture the authorized Pristine homepage over HTTP(S)." - ) - raise PrivateInputError( - "Benchmark snapshot sources must be HTTP(S) URLs. Local HTML files " - "cannot substitute for the Pristine homepage." - ) - - -def _load_snapshot( - html_path: Path, - metadata_path: Path, - source_url: str, -) -> BenchmarkSnapshot: - """Validate an existing snapshot without fetching or changing it.""" - if not html_path.is_file() or not metadata_path.is_file(): - raise PrivateInputError( - "The benchmark snapshot is incomplete. It will not be refetched or " - "overwritten; restore the matching HTML and metadata files." - ) - try: - metadata = SnapshotMetadata.from_json( - json.loads(metadata_path.read_text(encoding="utf-8")) - ) - except (OSError, UnicodeError, ValueError) as error: - raise PrivateInputError( - "The benchmark snapshot metadata is unreadable. The existing " - "snapshot will not be overwritten." - ) from error - if metadata.source_url != source_url: - raise PrivateInputError( - "The existing benchmark snapshot was captured from " - f"{metadata.source_url!r}, " - f"not {source_url!r}. It will not be refetched or overwritten." - ) - try: - body = html_path.read_bytes() - except OSError as error: - raise PrivateInputError( - "The benchmark snapshot HTML is unreadable. The existing snapshot " - "will not be overwritten." - ) from error - actual_sha256 = hashlib.sha256(body).hexdigest() - if metadata.sha256 != actual_sha256: - raise PrivateInputError( - "The benchmark snapshot checksum does not match its metadata. The " - "existing snapshot will not be refetched or overwritten." - ) - if metadata.byte_length != len(body): - raise PrivateInputError( - "The benchmark snapshot byte length does not match its metadata. The " - "existing snapshot will not be refetched or overwritten." - ) - if metadata.html_file != html_path.name: - raise PrivateInputError( - "The benchmark metadata identifies a different HTML file. The " - "existing snapshot will not be refetched or overwritten." - ) - return BenchmarkSnapshot(html_path, metadata_path, metadata, reused=True) - - -def _save_new_snapshot( - snapshot_directory: Path, - body: bytes, - metadata: SnapshotMetadata, -) -> None: - """Stage both artifacts and publish their directory as one atomic unit.""" - metadata_bytes = ( - json.dumps( - metadata.to_json(), - indent=2, - sort_keys=True, - ensure_ascii=False, - ) - + "\n" - ).encode("utf-8") - staged_directory = Path( - mkdtemp( - prefix=f".{SNAPSHOT_DIRECTORY_NAME}-", - dir=snapshot_directory.parent, - ) - ) - try: - (staged_directory / SNAPSHOT_HTML_FILENAME).write_bytes(body) - (staged_directory / SNAPSHOT_METADATA_FILENAME).write_bytes(metadata_bytes) - if snapshot_directory.exists(): - raise FileExistsError(snapshot_directory) - staged_directory.rename(snapshot_directory) - except OSError as error: - if snapshot_directory.exists(): - message = ( - "A benchmark snapshot appeared during capture. It was not " - "overwritten; rerun prepare to verify the existing snapshot." - ) - else: - message = f"Could not save the benchmark snapshot: {error}" - raise PrivateInputError(message) from error - finally: - if staged_directory.exists(): - for staged_file in staged_directory.iterdir(): - staged_file.unlink() - staged_directory.rmdir() diff --git a/vision_aid/evaluation/workspace.py b/vision_aid/evaluation/workspace.py deleted file mode 100644 index e6a9ff4..0000000 --- a/vision_aid/evaluation/workspace.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Private filesystem conventions for model evaluation artifacts.""" - -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from uuid import uuid4 - -DEFAULT_WORKBOOK_FILENAME = "Pristine Accessibility Defect Report.xlsx" -DEFAULT_WORKSPACE = Path(".model-evaluation") -WORKSPACE_PARTITIONS = ( - "references", - "reviews", - "snapshots", - "runs", - "reports", -) - - -class PrivateInputError(ValueError): - """Report a missing local input without attempting to replace it.""" - - -@dataclass(frozen=True) -class PrivateWorkspace: - """Paths reserved for private evaluation inputs and derived artifacts.""" - - root: Path - - @classmethod - def from_current_directory(cls) -> "PrivateWorkspace": - """Anchor artifacts at the worktree root or the non-versioned CWD.""" - current_directory = Path.cwd().resolve() - worktree = find_worktree_root(current_directory) - return cls((worktree or current_directory) / DEFAULT_WORKSPACE) - - @classmethod - def from_evaluation_run(cls, run_directory: Path) -> "PrivateWorkspace": - """Recover the private workspace associated with one planned run.""" - resolved_run = run_directory.resolve() - if resolved_run.parent.name != "runs": - raise PrivateInputError( - f"Evaluation run {resolved_run} is not inside a runs " - "partition." - ) - workspace = cls(resolved_run.parent.parent) - workspace.require_initialized() - return workspace - - @property - def partitions(self) -> tuple[Path, ...]: - """Return every required private artifact partition.""" - return tuple(self.root / name for name in WORKSPACE_PARTITIONS) - - def initialize(self) -> None: - """Create the private workspace and all of its artifact partitions.""" - for partition in self.partitions: - partition.mkdir(parents=True, exist_ok=True) - - def require_initialized(self) -> None: - """Fail with recovery instructions unless every partition exists.""" - missing = [path.name for path in self.partitions if not path.is_dir()] - if missing: - raise PrivateInputError( - f"Private evaluation workspace is not initialized at " - f"{self.root.resolve()}. Run 'visionaid-evaluate prepare' " - "with an authorized local workbook first." - ) - - def create_evaluation_run(self) -> tuple[str, Path]: - """Create and return one uniquely identified evaluation run.""" - self.require_initialized() - timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") - run_id = f"run-{timestamp}-{uuid4().hex[:12]}" - run_directory = self.root / "runs" / run_id - run_directory.mkdir(exist_ok=False) - return run_id, run_directory - - -def find_worktree_root(start: Path) -> Path | None: - """Find the containing Git worktree without invoking Git.""" - resolved_start = start.resolve() - for directory in (resolved_start, *resolved_start.parents): - if (directory / ".git").exists(): - return directory - return None - - -def require_private_workbook(path: Path) -> Path: - """Resolve a local workbook or explain how an authorized user supplies it.""" - candidate = path - current_worktree = find_worktree_root(Path.cwd()) - if path == Path(DEFAULT_WORKBOOK_FILENAME) and current_worktree is not None: - candidate = current_worktree / path - - if not candidate.is_file(): - raise PrivateInputError( - f"Private source workbook not found at {candidate.resolve()}. Obtain " - f"'{DEFAULT_WORKBOOK_FILENAME}' from an authorized project source " - "and place it at the repository root, or pass --workbook PATH. " - "The evaluation CLI will not download or substitute private inputs." - ) - workbook = candidate.resolve() - worktree = find_worktree_root(workbook.parent) - if worktree is None or not workbook.is_relative_to(worktree): - return workbook - - default_workbook = (worktree / DEFAULT_WORKBOOK_FILENAME).resolve() - private_workspace = (worktree / DEFAULT_WORKSPACE).resolve() - if workbook != default_workbook and not workbook.is_relative_to(private_workspace): - raise PrivateInputError( - f"Private source workbook at {workbook} would be visible to " - "version control. Keep the exact default filename at the repository " - "root, store the override inside .model-evaluation, or pass a path " - "outside the repository." - ) - return workbook From b32eea12d0dbff2db9f7d9ff003abb955547354b Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Wed, 23 Sep 2026 18:43:19 -0400 Subject: [PATCH 031/126] fix(evaluation): close review gaps in guarded runs --- processing_scripts/llm_client/audit.py | 29 +++- tests/test_audit_request.py | 111 +++--------- tests/test_evaluation_cli.py | 232 ++++++++++++++++++++----- vision_aid/evaluation/audit.py | 77 +++++--- vision_aid/evaluation/cli.py | 124 +++++++++---- vision_aid/evaluation/normalization.py | 33 +--- vision_aid/evaluation/pricing.py | 13 +- vision_aid/evaluation/references.py | 48 +++-- vision_aid/evaluation/review.py | 150 +++++++++++----- vision_aid/evaluation/schemas.py | 1 + 10 files changed, 539 insertions(+), 279 deletions(-) diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index d22cb19..ebc7315 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -6,8 +6,8 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum -from typing import Any, Literal from types import SimpleNamespace +from typing import Any, Literal from .client import is_gemini_model, is_openai_model, supports_temperature @@ -139,15 +139,21 @@ def _usage_result( class _StreamFailure(Exception): - """Carry observed usage across a transport failure without response content.""" + """Carry observed usage and final text across a transport failure.""" def __init__( - self, cause: Exception, usage: dict[str, Any], stop_reason: str | None + self, + cause: Exception, + usage: dict[str, Any], + stop_reason: str | None, + response_text: str, ): + """Retain partial evidence without retaining thinking blocks.""" super().__init__(str(cause)) self.cause = cause self.usage = usage self.stop_reason = stop_reason + self.response_text = response_text class AuditRequestClient: @@ -166,6 +172,7 @@ def __init__( *, request_config: AuditRequestConfig | None = None, provider_client: Any | None = None, + max_retries: int | None = None, ): """Resolve legacy arguments and initialize the selected SDK client.""" if request_config is None: @@ -202,7 +209,10 @@ def __init__( else: import anthropic - self._client = anthropic.Anthropic(api_key=api_key) + options: dict[str, Any] = {"api_key": api_key} + if max_retries is not None: + options["max_retries"] = max_retries + self._client = anthropic.Anthropic(**options) def call(self, prompt: str) -> dict[str, Any]: """Send one prompt and return a provider-neutral audit result.""" @@ -217,16 +227,18 @@ def call(self, prompt: str) -> dict[str, Any]: result = self._call_anthropic(prompt, start) result["request"] = resolved_request_metadata(self.request_config) return result - except Exception as exc: + except Exception as exc: # noqa: BLE001 - report provider failures as evidence partial_usage = {} stop_reason = None + response_text = None if isinstance(exc, _StreamFailure): partial_usage = exc.usage stop_reason = exc.stop_reason + response_text = exc.response_text exc = exc.cause return { "success": False, - "response": None, + "response": response_text, "model": self.model, "provider": self._provider_identity(), "request": resolved_request_metadata(self.request_config), @@ -355,7 +367,10 @@ def _stream_anthropic(self, request_kwargs: dict[str, Any]) -> Any: raise ValueError("Anthropic stream ended before message_stop") except Exception as exc: raise _StreamFailure( - exc, usage, message.stop_reason if message is not None else None + exc, + usage, + message.stop_reason if message is not None else None, + "".join(text_parts), ) from exc message.content = [SimpleNamespace(type="text", text="".join(text_parts))] message.usage = SimpleNamespace(**usage) diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index ef80573..d412a99 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -69,90 +69,6 @@ def __init__(self, response): class AuditRequestTests(TestCase): - def test_luna_request_uses_medium_reasoning_without_temperature(self): - """Luna uses the explicit evaluation request shape.""" - response = SimpleNamespace( - id="chatcmpl-eval-1", - model="gpt-5.6-luna", - service_tier="default", - system_fingerprint="fp_test", - choices=[ - SimpleNamespace( - message=SimpleNamespace(content='{"findings": []}'), - finish_reason="stop", - ) - ], - usage=SimpleNamespace( - prompt_tokens=120, - completion_tokens=30, - total_tokens=150, - prompt_tokens_details=SimpleNamespace(cached_tokens=20), - completion_tokens_details=SimpleNamespace(reasoning_tokens=10), - ), - ) - fake = OpenAIFake(response) - config = AuditRequestConfig( - model="gpt-5.6-luna", - reasoning_effort="medium", - temperature=None, - max_output_tokens=8192, - ) - - client = PipelineClient( - api_key="unused-by-fake", - request_config=config, - provider_client=fake, - ) - result = client.call("audit this payload") - - self.assertEqual( - fake.chat.completions.requests, - [ - { - "model": "gpt-5.6-luna", - "max_completion_tokens": 8192, - "reasoning_effort": "medium", - "messages": [{"role": "user", "content": "audit this payload"}], - } - ], - ) - self.assertTrue(result["success"]) - self.assertEqual( - result["provider"], - { - "name": "openai", - "endpoint": "chat.completions", - "response_id": "chatcmpl-eval-1", - "response_model": "gpt-5.6-luna", - "service_tier": "default", - "system_fingerprint": "fp_test", - }, - ) - self.assertEqual(result["usage"]["input_tokens"], 120) - self.assertEqual(result["usage"]["output_tokens"], 30) - self.assertEqual(result["usage"]["total_tokens"], 150) - self.assertEqual(result["usage"]["cached_input_tokens"], 20) - self.assertEqual(result["usage"]["reasoning_tokens"], 10) - self.assertEqual( - result["usage"]["provider_usage"]["prompt_tokens_details"], - {"cached_tokens": 20}, - ) - self.assertEqual(result["stop_reason"], "stop") - self.assertIsInstance(result["duration_seconds"], float) - self.assertIsNone(result["failure"]) - self.assertEqual( - result["request"], - { - "model": "gpt-5.6-luna", - "reasoning_effort": "medium", - "max_output_tokens": 8192, - "sampling": { - "temperature": None, - "omitted_parameters": ["temperature"], - }, - }, - ) - def test_legacy_provider_request_defaults_are_preserved(self): """Implicit configuration preserves all production request defaults.""" anthropic_response = SimpleNamespace( @@ -347,7 +263,9 @@ def test_pipeline_persists_explicit_request_configuration(self): "parts,stop", [(["[", "]"], "end_turn"), ([], "end_turn"), (["["], "max_tokens")] ) def test_haiku_thinking_stream_exposes_only_final_text(parts, stop): + """Only final text and inclusive billed output leave a thinking stream.""" from contextlib import nullcontext + from processing_scripts.llm_client.audit import AuditRequestClient def event(kind, **values): @@ -416,7 +334,9 @@ def event(kind, **values): def test_interrupted_thinking_stream_keeps_reported_billed_usage(): + """An interrupted stream retains text already received and billed usage.""" from contextlib import nullcontext + from processing_scripts.llm_client.audit import AuditRequestClient def events(): @@ -429,6 +349,10 @@ def events(): usage=SimpleNamespace(input_tokens=100, output_tokens=1), ), ) + yield SimpleNamespace( + type="content_block_start", + content_block=SimpleNamespace(type="text", text="partial finding"), + ) yield SimpleNamespace( type="message_delta", delta=SimpleNamespace(stop_reason="max_tokens"), @@ -447,9 +371,26 @@ def events(): ), ).call("audit") assert not result["success"] - assert result["response"] is None + assert result["response"] == "partial finding" assert result["usage"]["input_tokens"] == 100 assert result["usage"]["output_tokens"] == 16002 assert result["usage"]["reasoning_tokens"] is None assert result["stop_reason"] == "max_tokens" assert result["failure"]["type"] == "TimeoutError" + + +def test_evaluation_can_disable_anthropic_sdk_retries(monkeypatch): + """The request client forwards a zero retry policy only when requested.""" + from processing_scripts.llm_client.audit import AuditRequestClient + + options = [] + monkeypatch.setattr( + "anthropic.Anthropic", + lambda **kwargs: options.append(kwargs) or AnthropicFake(None), + ) + AuditRequestClient( + "unused", + request_config=AuditRequestConfig(model="claude-haiku-4-5-20251001"), + max_retries=0, + ) + assert options == [{"api_key": "unused", "max_retries": 0}] diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 22b331d..0e150f5 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -13,43 +13,114 @@ def inputs(tmp_path: Path) -> tuple[Path, Path]: + """Build a small workbook with the exact supported source columns.""" book = openpyxl.Workbook() sheet = book.active sheet.title = "Defect report" - sheet.append(["Sr. #", "element name", "Browser Combination", "Page name", - "Issue Title", "Steps to Reproduce", "actual result", - "expected result", "Recommendation for Fix", "WCAG Sc", - "Type of Change", "comment"]) - sheet.append([1, "First link", "Chrome", "Home", "Unclear link", "Open page", - "Vague link\nfor everyone", "Descriptive text", "Rename", "2.4.4", "Code", ""]) - sheet.append([2, "Second link", "Chrome", "Global", "Missing destination", - "Open page", "No href", "Working link", "Add href", "2.4.4", "Code", ""]) - sheet.append([3, "Photo", "Chrome", "Home", "No alt text", "Open page", - "Image lacks alt", "Alt text", "Add alt", "1.1.1", "Code", ""]) + sheet.append( + [ + "Sr. #", + "element name", + "Browser Combination", + "Page name", + "Issue Title", + "Steps to Reproduce", + "actual result", + "expected result", + "Recommendation for Fix", + "WCAG Sc", + "Type of Change", + "comment", + ] + ) + sheet.append( + [ + 1, + "First link", + "Chrome", + "Home", + "Unclear link", + "Open page", + "Vague link\nfor everyone", + "Descriptive text", + "Rename", + "2.4.4", + "Code", + "", + ] + ) + sheet.append( + [ + 2, + "Second link", + "Chrome", + "Global", + "Missing destination", + "Open page", + "No href", + "Working link", + "Add href", + "2.4.4", + "Code", + "", + ] + ) + sheet.append( + [ + 3, + "Photo", + "Chrome", + "Home", + "No alt text", + "Open page", + "Image lacks alt", + "Alt text", + "Add alt", + "1.1.1", + "Code", + "", + ] + ) sheet.append([4, "Other", "Chrome", "Careers", "Outside scope"]) workbook = tmp_path / "reference.xlsx" book.save(workbook) html = tmp_path / "home.html" - html.write_text('Home' - '

Home

Click here') + html.write_text( + 'Home' + '

Home

Click here' + ) return workbook, html def args(workbook: Path, html: Path, *extra: str) -> list[str]: - return ["run", "--workbook", str(workbook), "--html", str(html), - "--source-url", "https://example.test/", "--max-cost-usd", "0.01", *extra] + """Assemble the public run command for the synthetic inputs.""" + return [ + "run", + "--workbook", + str(workbook), + "--html", + str(html), + "--source-url", + "https://example.test/", + "--max-cost-usd", + "0.01", + *extra, + ] def directory(tmp_path: Path) -> Path: + """Find the single run created under a temporary workspace.""" return next((tmp_path / ".model-evaluation" / "runs").iterdir()) def read_review(path: Path) -> list[dict[str, str]]: + """Read reviewer decisions using standard CSV quoting.""" with path.open(newline="") as stream: return list(csv.DictReader(stream)) def write_review(path: Path, rows: list[dict[str, str]]) -> None: + """Save edited reviewer decisions using standard CSV quoting.""" with path.open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=rows[0]) writer.writeheader() @@ -59,10 +130,13 @@ def write_review(path: Path, rows: list[dict[str, str]]) -> None: def test_preview_preserves_reference_rows_and_never_calls_provider( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: + """Credentials alone never buy requests, and all source rows survive.""" workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "available") - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: pytest.fail("paid call")) + monkeypatch.setattr( + audit, "AuditRequestClient", lambda **_: pytest.fail("paid call") + ) assert main(args(workbook, html)) == 0 output = capsys.readouterr().out run = directory(tmp_path) @@ -80,9 +154,24 @@ def test_preview_preserves_reference_rows_and_never_calls_provider( main(["report", "--run-dir", str(run)]) +def test_subdirectory_invocation_uses_ignored_worktree_run_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A run from a repository subdirectory stays under the root ignore rule.""" + workbook, html = inputs(tmp_path) + (tmp_path / ".git").mkdir() + child = tmp_path / "entry_points" + child.mkdir() + monkeypatch.chdir(child) + assert main(args(workbook, html)) == 0 + assert (tmp_path / ".model-evaluation" / "runs").is_dir() + assert not (child / ".model-evaluation").exists() + + def test_live_csv_review_and_report_full_row_metrics( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """One fake live run yields hand-calculated metrics and source evidence.""" workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -91,28 +180,58 @@ class FakeClient: calls = 0 def call(self, prompt: str) -> dict: + """Return a successful fake provider response with usage.""" self.calls += 1 - return {"success": True, "response": json.dumps([{"problem": "Unclear link", - "location": "First link"}]), "usage": {"input_tokens": 100, - "output_tokens": 20, "cached_input_tokens": 50, - "cache_creation_input_tokens": 10, "reasoning_tokens": None}} + return { + "success": True, + "response": json.dumps( + [{"problem": "Unclear link", "location": "First link"}] + ), + "usage": { + "input_tokens": 100, + "output_tokens": 20, + "cached_input_tokens": 50, + "cache_creation_input_tokens": 10, + "reasoning_tokens": None, + }, + } fake = FakeClient() - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: fake) + client_options: list[dict] = [] + + def client(**options: object) -> FakeClient: + """Record the evaluator's request-client configuration.""" + client_options.append(options) + return fake + + monkeypatch.setattr(audit, "AuditRequestClient", client) assert main(args(workbook, html, "--live", "--approve-live")) == 0 + assert client_options[0]["max_retries"] == 0 run = directory(tmp_path) assert fake.calls == 3 - assert json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" + assert ( + json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" + ) findings = json.loads((run / "audit-findings.json").read_text()) programmatic = json.loads((run / "programmatic-findings.json").read_text()) assert findings and programmatic review = run / "review.csv" rows = read_review(review) - rows[0].update(classification="llm_eligible", classification_reason="Model can see link", - audit_finding_id=findings[0]["finding_id"], match_reason="Full issue") - rows[1].update(classification="programmatic", classification_reason="HTML check", - programmatic_finding_id=programmatic[0]["finding_id"], match_reason="Full issue") - rows[2].update(classification="unavailable_evidence", classification_reason="Visual context") + rows[0].update( + classification="llm_eligible", + classification_reason="Model can see link", + audit_finding_id=findings[0]["finding_id"], + match_reason="Full issue", + ) + rows[1].update( + classification="programmatic", + classification_reason="HTML check", + programmatic_finding_id=programmatic[0]["finding_id"], + match_reason="Full issue", + ) + rows[2].update( + classification="unavailable_evidence", classification_reason="Visual context" + ) write_review(review, rows) assert main(["report", "--run-dir", str(run)]) == 0 report = (run / "report.md").read_text() @@ -123,33 +242,46 @@ def call(self, prompt: str) -> dict: def test_review_requires_classification_and_rejects_invalid_finding_ids( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: + """Review validation blocks missing decisions, wrong kinds, and reuse.""" workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") class FakeClient: def call(self, prompt: str) -> dict: - return {"success": True, "response": json.dumps([{"problem": "Vague link"}]), - "usage": {"input_tokens": 1, "output_tokens": 1}} + """Return one normalized finding for applicable prompts.""" + return { + "success": True, + "response": json.dumps([{"problem": "Vague link"}]), + "usage": {"input_tokens": 1, "output_tokens": 1}, + } monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: FakeClient()) assert main(args(workbook, html, "--live", "--approve-live")) == 0 run = directory(tmp_path) review = run / "review.csv" - audit_ids = [item["finding_id"] for item in json.loads( - (run / "audit-findings.json").read_text())] - programmatic_id = json.loads( - (run / "programmatic-findings.json").read_text())[0]["finding_id"] + audit_ids = [ + item["finding_id"] + for item in json.loads((run / "audit-findings.json").read_text()) + ] + programmatic_id = json.loads((run / "programmatic-findings.json").read_text())[0][ + "finding_id" + ] rows = read_review(review) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) - rows[0].update(classification="llm_eligible", classification_reason="Visible", - audit_finding_id="; ".join(audit_ids[:2]), - programmatic_finding_id=programmatic_id, - match_reason="Together cover the row", review_notes="Partial alone") + rows[0].update( + classification="llm_eligible", + classification_reason="Visible", + audit_finding_id="; ".join(audit_ids[:2]), + programmatic_finding_id=programmatic_id, + match_reason="Together cover the row", + review_notes="Partial alone", + ) rows[1].update(classification="programmatic", classification_reason="Markup") rows[2].update(classification="ambiguous", classification_reason="Unclear claim") write_review(review, rows) @@ -180,31 +312,40 @@ def call(self, prompt: str) -> dict: rows[2]["classification"] = "unavailable_evidence" write_review(review, rows) assert main(["report", "--run-dir", str(run)]) == 0 - assert "Workbook-row recall: N/A (0 eligible rows)" in (run / "report.md").read_text() + assert ( + "Workbook-row recall: N/A (0 eligible rows)" in (run / "report.md").read_text() + ) assert "Combined workbook coverage: 0/3" in (run / "report.md").read_text() def test_live_execution_needs_explicit_approval( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """A noninteractive live flag without approval makes no paid request.""" workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "available") - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: pytest.fail("paid call")) + monkeypatch.setattr( + audit, "AuditRequestClient", lambda **_: pytest.fail("paid call") + ) class DeclinedInput: def isatty(self) -> bool: + """Represent an unattended input stream.""" return False monkeypatch.setattr("sys.stdin", DeclinedInput()) assert main(args(workbook, html, "--live")) == 0 - assert json.loads((directory(tmp_path) / "run.json").read_text())["complete"] is False + assert ( + json.loads((directory(tmp_path) / "run.json").read_text())["complete"] is False + ) @pytest.mark.parametrize("failure", [True, False]) def test_failed_or_over_budget_runs_keep_usage_and_cannot_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool ) -> None: + """Failure and budget exhaustion stop after one request and remain inspectable.""" workbook, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -213,11 +354,14 @@ class FakeClient: calls = 0 def call(self, prompt: str) -> dict: + """Return either a failure or an expensive successful response.""" self.calls += 1 - return {"success": not failure, - "response": "[]" if not failure else None, - "error": "synthetic failure" if failure else None, - "usage": {"input_tokens": 100000, "output_tokens": 20}} + return { + "success": not failure, + "response": "[]" if not failure else None, + "error": "synthetic failure" if failure else None, + "usage": {"input_tokens": 100000, "output_tokens": 20}, + } fake = FakeClient() monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: fake) diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index 5897736..50659d4 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -14,26 +14,38 @@ MODEL = "claude-haiku-4-5-20251001" REQUEST_CONFIG = AuditRequestConfig( - model=MODEL, thinking_budget_tokens=16000, - max_output_tokens=24192, temperature=None, + model=MODEL, + thinking_budget_tokens=16000, + max_output_tokens=24192, + temperature=None, ) CONFIGURATION = { - "model": MODEL, "provider": "anthropic", "endpoint": "messages", - "thinking_budget_tokens": 16000, "max_output_tokens": 24192, - "temperature": "omitted", "summaries": "disabled", + "model": MODEL, + "provider": "anthropic", + "endpoint": "messages", + "thinking_budget_tokens": REQUEST_CONFIG.thinking_budget_tokens, + "max_output_tokens": REQUEST_CONFIG.max_output_tokens, + "temperature": "omitted", + "summaries": "disabled", } USAGE_KEYS = ( - "input_tokens", "cached_input_tokens", "cache_creation_input_tokens", + "input_tokens", + "cached_input_tokens", + "cache_creation_input_tokens", "output_tokens", ) def write_json(path: Path, value: object) -> None: - path.write_text(json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True) + "\n", - encoding="utf-8") + """Save one readable private evidence artifact.""" + path.write_text( + json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) def positive_cost(value: str) -> Decimal: + """Parse a strictly positive cost guardrail.""" try: amount = Decimal(value) except Exception as error: @@ -46,7 +58,11 @@ def positive_cost(value: str) -> Decimal: def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: """Make each paid request once, saving reported usage and responses as received.""" schedule = PriceSchedule.load(run / "pricing.json") - client = AuditRequestClient(api_key=api_key, request_config=REQUEST_CONFIG) + client = AuditRequestClient( + api_key=api_key, + request_config=REQUEST_CONFIG, + max_retries=0, + ) usage: dict[str, int] = {key: 0 for key in USAGE_KEYS} responses: list[dict[str, Any]] = [] findings: list[dict[str, Any]] = [] @@ -54,40 +70,59 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: complete = True reason = None for prompt in manifest["prompts"]: - if Decimal(schedule.estimate(MODEL, usage)) >= Decimal(manifest["max_cost_usd"]): + if Decimal(schedule.estimate(MODEL, usage)) >= Decimal( + manifest["max_cost_usd"] + ): complete, reason = False, "cost guardrail exhausted" break name = prompt["name"] - prompt_text = json.loads((run / "prompts" / f"{name}.json").read_text())["prompt_text"] + prompt_text = json.loads((run / "prompts" / f"{name}.json").read_text())[ + "prompt_text" + ] result: dict[str, Any] try: result = client.call(prompt_text) - except Exception as error: - result = {"success": False, "response": None, "usage": {}, - "error": str(error)} + except Exception as error: # noqa: BLE001 + # Retain an unexpected request failure as run evidence. + result = { + "success": False, + "response": None, + "usage": {}, + "error": str(error), + } reported: dict[str, Any] = result.get("usage") or {} for key in USAGE_KEYS: usage[key] += int(reported.get(key) or 0) responses.append({"prompt": name, "result": result}) if result.get("success"): normalized = normalize_prompt_response( - name, str(result.get("response") or ""), run_id=manifest["run_id"], - model=MODEL, page_url=manifest["source_url"], + name, + str(result.get("response") or ""), + run_id=manifest["run_id"], + model=MODEL, + page_url=manifest["source_url"], ) findings.extend(asdict(finding) for finding in normalized.findings) if normalized.parse_status == "malformed": format_failures.append(name) else: complete, reason = False, str(result.get("error") or "request failed") - manifest.update(usage=usage, estimated_cost_usd=schedule.estimate(MODEL, usage), - complete=False, incomplete_reason=reason, - format_failures=format_failures) + manifest.update( + usage=usage, + estimated_cost_usd=schedule.estimate(MODEL, usage), + complete=False, + incomplete_reason=reason, + format_failures=format_failures, + ) write_json(run / "raw-responses.json", responses) write_json(run / "audit-findings.json", findings) write_json(run / "run.json", manifest) if not complete: break - manifest.update(complete=complete, incomplete_reason=reason, - estimated_cost_usd=schedule.estimate(MODEL, usage)) + manifest.update( + complete=complete, + incomplete_reason=reason, + estimated_cost_usd=schedule.estimate(MODEL, usage), + ) write_json(run / "run.json", manifest) return complete diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index c5b436c..1c64cd5 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -18,7 +18,12 @@ from entry_points.run_pipeline import run_pipeline from vision_aid.evaluation.audit import ( - CONFIGURATION, MODEL, REQUEST_CONFIG, execute, positive_cost, write_json, + CONFIGURATION, + MODEL, + REQUEST_CONFIG, + execute, + positive_cost, + write_json, ) from vision_aid.evaluation.normalization import normalize_programmatic_findings from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule @@ -30,23 +35,43 @@ def build_parser() -> argparse.ArgumentParser: + """Expose the two supported evaluation operations.""" parser = argparse.ArgumentParser(prog="visionaid-evaluate") commands = parser.add_subparsers(dest="command", required=True) - run = commands.add_parser("run", help="preview or execute the fixed Haiku experiment") + run = commands.add_parser( + "run", help="preview or execute the fixed Haiku experiment" + ) run.add_argument("--workbook", type=Path, default=DEFAULT_WORKBOOK) run.add_argument("--html", type=Path, required=True, help="local Benchmark HTML") run.add_argument("--source-url", required=True, help="URL represented by the HTML") - run.add_argument("--max-cost-usd", required=True, help="positive between-request limit") + run.add_argument( + "--max-cost-usd", required=True, help="positive between-request limit" + ) run.add_argument("--live", action="store_true", help="enable billable execution") - run.add_argument("--approve-live", action="store_true", - help="explicitly approve billable execution without a prompt") - report = commands.add_parser("report", help="score completed run and edited review.csv") + run.add_argument( + "--approve-live", + action="store_true", + help="explicitly approve billable execution without a prompt", + ) + report = commands.add_parser( + "report", help="score completed run and edited review.csv" + ) report.add_argument("--run-dir", type=Path, required=True) return parser def _new_run_directory() -> Path: - root = Path.cwd() / ".model-evaluation" / "runs" + """Create a unique private directory under the containing worktree root.""" + current = Path.cwd() + worktree = next( + ( + parent + for parent in (current, *current.parents) + if (parent / ".git").exists() + ), + current, + ) + root = worktree / ".model-evaluation" / "runs" root.mkdir(parents=True, exist_ok=True) run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8] run = root / run_id @@ -55,6 +80,7 @@ def _new_run_directory() -> Path: def _run(args: argparse.Namespace) -> int: + """Prepare a fresh run and execute only after explicit approval.""" limit = positive_cost(args.max_cost_usd) if args.approve_live and not args.live: raise ValueError("--approve-live requires --live") @@ -67,42 +93,69 @@ def _run(args: argparse.Namespace) -> int: snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() write_json(run / "references.json", references.to_dict()) production = run_pipeline( - html_path=str(run / "snapshot.html"), output_dir=run, api_key=None, - model=MODEL, dry_run=True, include_summaries=False, + html_path=str(run / "snapshot.html"), + output_dir=run, + api_key=None, + model=MODEL, + dry_run=True, + include_summaries=False, request_config=REQUEST_CONFIG, ) (run / "manifest.json").unlink() raw_programmatic = json.loads((run / "programmatic_findings.json").read_text()) findings = normalize_programmatic_findings( - raw_programmatic, run_id=run.name, page_url=args.source_url, + raw_programmatic, + run_id=run.name, + page_url=args.source_url, ) write_json(run / "programmatic-findings.json", [asdict(item) for item in findings]) create_review(run / "review.csv", list(references.to_dict()["references"])) schedule = PriceSchedule.load(run / "pricing.json") manifest = { - "run_id": run.name, "source_url": args.source_url, + "run_id": run.name, + "source_url": args.source_url, "workbook_filename": references.workbook_filename, "workbook_sha256": references.workbook_sha256, "snapshot_sha256": snapshot_hash, "configuration": CONFIGURATION, - "pricing_version": schedule.version, "pricing_sha256": schedule.identity, - "max_cost_usd": str(limit), "complete": False, - "incomplete_reason": "preview", "estimated_cost_usd": "0", - "usage": {key: 0 for key in ( - "input_tokens", "cached_input_tokens", - "cache_creation_input_tokens", "output_tokens", - )}, + "pricing_version": schedule.version, + "pricing_sha256": schedule.identity, + "max_cost_usd": str(limit), + "complete": False, + "incomplete_reason": "preview", + "estimated_cost_usd": "0", + "usage": { + key: 0 + for key in ( + "input_tokens", + "cached_input_tokens", + "cache_creation_input_tokens", + "output_tokens", + ) + }, "format_failures": [], - "prompts": [{"name": entry["name"], "checklist": entry["checklist"]} - for entry in production["prompts_dry_run"]], + "prompts": [ + {"name": entry["name"], "checklist": entry["checklist"]} + for entry in production["prompts_dry_run"] + ], "filters": production["pass1_filters_active"], "skipped_prompts": production["prompts_skipped"], } write_json(run / "run.json", manifest) print(f"Evaluation run: {run}") - print(f"Model: {MODEL}; endpoint: Messages; thinking: 16000; output cap: 24192") - print(f"Temperature: omitted; summaries: disabled; prompts: {len(manifest['prompts'])}") - print(f"Snapshot SHA-256: {snapshot_hash}; Reference rows: {len(references.references)}") + print( + f"Model: {MODEL}; endpoint: Messages; " + f"thinking: {CONFIGURATION['thinking_budget_tokens']}; " + f"output cap: {CONFIGURATION['max_output_tokens']}" + ) + print( + "Temperature: omitted; summaries: disabled; " + f"prompts: {len(manifest['prompts'])}" + ) + print( + f"Snapshot SHA-256: {snapshot_hash}; " + f"Reference rows: {len(references.references)}" + ) print(f"Pricing: {schedule.version} ({schedule.identity})") print(f"Cost guardrail: ${limit}; checked before each request") print(f"Review CSV: {run / 'review.csv'}") @@ -113,16 +166,19 @@ def _run(args: argparse.Namespace) -> int: api_key = os.getenv("ANTHROPIC_API_KEY") if not api_key: raise ValueError("Live execution requires ANTHROPIC_API_KEY") - if not args.approve_live: - if not sys.stdin.isatty() or input("Type 'yes' to approve paid execution: ") != "yes": - print("Live execution declined; no paid requests made.") - return 0 + if not args.approve_live and ( + not sys.stdin.isatty() + or input("Type 'yes' to approve paid execution: ") != "yes" + ): + print("Live execution declined; no paid requests made.") + return 0 complete = execute(run, manifest, api_key) print(f"Run {'complete' if complete else 'incomplete'}: {run}") return 0 if complete else 1 def _report(args: argparse.Namespace) -> int: + """Validate human decisions and report a completed run.""" run = args.run_dir.resolve() manifest = json.loads((run / "run.json").read_text()) if manifest["run_id"] != run.name: @@ -135,17 +191,23 @@ def _report(args: argparse.Namespace) -> int: audit = json.loads((run / "audit-findings.json").read_text()) programmatic = json.loads((run / "programmatic-findings.json").read_text()) for kind, collection in (("audit", audit), ("programmatic", programmatic)): - if any(item["run_id"] != run.name or item["source"] != kind or - item["page_url"] != manifest["source_url"] for item in collection): + if any( + item["run_id"] != run.name + or item["source"] != kind + or item["page_url"] != manifest["source_url"] + for item in collection + ): raise ValueError(f"{kind} findings do not belong to this run") - rows = reviewed_rows(run / "review.csv", reference_set["references"], - audit, programmatic) + rows = reviewed_rows( + run / "review.csv", reference_set["references"], audit, programmatic + ) write_report(run / "report.md", manifest, rows, audit, programmatic) print(f"Markdown report: {run / 'report.md'}") return 0 def main(argv: Sequence[str] | None = None) -> int: + """Dispatch the requested CLI operation.""" parser = build_parser() args = parser.parse_args(argv) try: diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index 81f94fb..a5fb14f 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -94,9 +94,7 @@ def _items(parsed: Any, spec: PromptSpec) -> list[dict[str, Any]] | None: return None for key in ("findings", "violations", "results"): nested = parsed.get(key) - if isinstance(nested, list) and all( - isinstance(item, dict) for item in nested - ): + if isinstance(nested, list) and all(isinstance(item, dict) for item in nested): return [item for item in nested if isinstance(item, dict)] return [parsed] if parsed else [] @@ -113,10 +111,7 @@ def _is_failure(prompt: str, item: dict[str, Any]) -> bool: if _has_values(item.get("issues")): return True if prompt == "page_title": - return ( - item.get("is_descriptive") is False - or item.get("matches_h1") is False - ) + return item.get("is_descriptive") is False or item.get("matches_h1") is False if prompt == "heading_structure": return item.get("structure_clear") is False or _has_values( item.get("vague_headings") @@ -164,8 +159,7 @@ def _is_failure(prompt: str, item: dict[str, Any]) -> bool: return item.get("pattern") in {"unlabeled_control", "missing_label"} if prompt == "media_captions": return ( - item.get("has_captions_track") is False - or item.get("has_controls") is False + item.get("has_captions_track") is False or item.get("has_controls") is False ) return False @@ -197,9 +191,7 @@ def normalize_prompt_response( parse_status="unknown_prompt", raw_response=raw_response, findings=(), - error=( - f"No fixed evaluation normalizer exists for {prompt_name!r}." - ), + error=(f"No fixed evaluation normalizer exists for {prompt_name!r}."), ) try: parsed = json.loads(_strip_fence(raw_response)) @@ -230,13 +222,10 @@ def normalize_prompt_response( sort_keys=True, separators=(",", ":"), ) - identity = ( - f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() - ) + identity = f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() findings.append( CanonicalFinding( - finding_id="finding-" - + hashlib.sha256(identity).hexdigest()[:20], + finding_id="finding-" + hashlib.sha256(identity).hexdigest()[:20], source="audit", run_id=run_id, model=model, @@ -275,14 +264,11 @@ def normalize_programmatic_findings( sort_keys=True, separators=(",", ":"), ) - identity = ( - f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() - ) + identity = f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() rule = str(item.get("rule_id") or item.get("rule") or "programmatic") normalized.append( CanonicalFinding( - finding_id="programmatic-" - + hashlib.sha256(identity).hexdigest()[:20], + finding_id="programmatic-" + hashlib.sha256(identity).hexdigest()[:20], source="programmatic", run_id=run_id, model=None, @@ -290,8 +276,7 @@ def normalize_programmatic_findings( checklist=str(item.get("checklist") or "programmatic"), page_url=page_url, problem_family=rule, - problem=_first_text(item, _PROBLEM_KEYS + ("message",)) - or rule, + problem=_first_text(item, _PROBLEM_KEYS + ("message",)) or rule, element=_first_text(item, _ELEMENT_KEYS), location=_first_text(item, _LOCATION_KEYS), wcag_evidence=_wcag(item, ()), diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index 2ca36c9..f3ba799 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -26,9 +26,7 @@ def load(cls, path: Path) -> PriceSchedule: """Load and content-address one JSON price schedule.""" raw = path.read_bytes() data = json.loads(raw) - if not isinstance(data, dict) or not isinstance( - data.get("models"), dict - ): + if not isinstance(data, dict) or not isinstance(data.get("models"), dict): raise TypeError("Price schedule must contain a models object.") return cls(data=data, identity=hashlib.sha256(raw).hexdigest()) @@ -57,13 +55,10 @@ def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: ) output_tokens = Decimal(usage.get("output_tokens", 0) or 0) cost = ( - input_tokens - * Decimal(rates["input_per_million_usd"]) - + cached_tokens - * Decimal(rates["cached_input_per_million_usd"]) + input_tokens * Decimal(rates["input_per_million_usd"]) + + cached_tokens * Decimal(rates["cached_input_per_million_usd"]) + cache_creation_tokens * Decimal(rates["cache_creation_input_per_million_usd"]) - + output_tokens - * Decimal(rates["output_per_million_usd"]) + + output_tokens * Decimal(rates["output_per_million_usd"]) ) / MILLION return format(cost.normalize(), "f") diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/references.py index fb4aeaf..fbfc026 100644 --- a/vision_aid/evaluation/references.py +++ b/vision_aid/evaluation/references.py @@ -12,12 +12,24 @@ SHEET = "Defect report" HEADERS = ( - "Sr. #", "element name", "Browser Combination", "Page name", - "Issue Title", "Steps to Reproduce", "actual result", "expected result", - "Recommendation for Fix", "WCAG Sc", "Type of Change", "comment", + "Sr. #", + "element name", + "Browser Combination", + "Page name", + "Issue Title", + "Steps to Reproduce", + "actual result", + "expected result", + "Recommendation for Fix", + "WCAG Sc", + "Type of Change", + "comment", ) ELIGIBILITY_VALUES = ( - "llm_eligible", "programmatic", "unavailable_evidence", "ambiguous", + "llm_eligible", + "programmatic", + "unavailable_evidence", + "ambiguous", ) @@ -29,13 +41,14 @@ def import_homepage_references(workbook_path: Path, homepage_url: str) -> Refere sheet = book[SHEET] rows = sheet.iter_rows(values_only=True) headers = tuple(str(value or "") for value in next(rows)) - if headers[:len(HEADERS)] != HEADERS: + if headers[: len(HEADERS)] != HEADERS: raise ValueError("Reference workbook has unexpected Defect report columns") references = [] for number, values in enumerate(rows, start=2): raw = { header: "" if value is None else str(value) - for header, value in zip(headers, values, strict=False) if header + for header, value in zip(headers, values, strict=False) + if header } scope = raw["Page name"].strip().casefold() if scope not in {"home", "global"}: @@ -43,15 +56,20 @@ def import_homepage_references(workbook_path: Path, homepage_url: str) -> Refere if not raw["Issue Title"].strip(): raise ValueError(f"Defect report row {number} has no Issue Title") identity = f"{checksum}:{SHEET}:{number}".encode() - references.append(ReferenceDefect( - reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], - source_sheet=SHEET, source_row=number, page_scope=scope, - source_element=raw["element name"], problem=raw["Issue Title"], - raw_evidence=raw, - wcag_evidence=tuple(dict.fromkeys( - re.findall(r"\b\d+\.\d+\.\d+\b", raw["WCAG Sc"]) - )), - )) + references.append( + ReferenceDefect( + reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], + source_sheet=SHEET, + source_row=number, + page_scope=scope, + source_element=raw["element name"], + problem=raw["Issue Title"], + raw_evidence=raw, + wcag_evidence=tuple( + dict.fromkeys(re.findall(r"\b\d+\.\d+\.\d+\b", raw["WCAG Sc"])) + ), + ) + ) finally: book.close() if not references: diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index e46c36a..743e7df 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -10,29 +10,41 @@ from vision_aid.evaluation.references import ELIGIBILITY_VALUES, HEADERS DECISION_COLUMNS = ( - "classification", "classification_reason", "audit_finding_id", - "programmatic_finding_id", "match_reason", "review_notes", + "classification", + "classification_reason", + "audit_finding_id", + "programmatic_finding_id", + "match_reason", + "review_notes", ) REVIEW_COLUMNS = ( - "reference_id", "source_sheet", "source_row", "page_scope", - "source_element", *HEADERS, *DECISION_COLUMNS, + "reference_id", + "source_sheet", + "source_row", + "page_scope", + "source_element", + *HEADERS, + *DECISION_COLUMNS, ) def create_review(path: Path, references: list[dict[str, Any]]) -> None: + """Create one editable row per Reference defect with blank decisions.""" with path.open("w", encoding="utf-8", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=REVIEW_COLUMNS) writer.writeheader() for item in references: - writer.writerow({ - "reference_id": item["reference_id"], - "source_sheet": item["source_sheet"], - "source_row": item["source_row"], - "page_scope": item["page_scope"], - "source_element": item["source_element"], - **item["raw_evidence"], - **{column: "" for column in DECISION_COLUMNS}, - }) + writer.writerow( + { + "reference_id": item["reference_id"], + "source_sheet": item["source_sheet"], + "source_row": item["source_row"], + "page_scope": item["page_scope"], + "source_element": item["source_element"], + **item["raw_evidence"], + **{column: "" for column in DECISION_COLUMNS}, + } + ) def _ids(value: str) -> tuple[str, ...]: @@ -44,8 +56,10 @@ def _ids(value: str) -> tuple[str, ...]: def reviewed_rows( - path: Path, references: list[dict[str, Any]], - audit: list[dict[str, Any]], programmatic: list[dict[str, Any]], + path: Path, + references: list[dict[str, Any]], + audit: list[dict[str, Any]], + programmatic: list[dict[str, Any]], ) -> list[dict[str, Any]]: """Validate completeness and finding kind before granting any row credit.""" with path.open(encoding="utf-8-sig", newline="") as stream: @@ -71,7 +85,9 @@ def reviewed_rows( reference = expected[reference_id] for field in ("source_sheet", "source_row", "source_element"): if str(row[field]) != str(reference[field]): - raise ValueError(f"Reference identity changed for {reference_id}: {field}") + raise ValueError( + f"Reference identity changed for {reference_id}: {field}" + ) classification = row["classification"].strip() if classification not in ELIGIBILITY_VALUES: raise ValueError(f"Reference {reference_id} needs a valid classification") @@ -79,78 +95,126 @@ def reviewed_rows( raise ValueError(f"Reference {reference_id} needs a classification reason") matched: dict[str, tuple[str, ...]] = {} for kind, column in ( - ("audit", "audit_finding_id"), ("programmatic", "programmatic_finding_id") + ("audit", "audit_finding_id"), + ("programmatic", "programmatic_finding_id"), ): ids = _ids(row[column]) unknown = set(ids) - set(by_kind[kind]) if unknown: - raise ValueError(f"Unknown {kind} finding ID: {', '.join(sorted(unknown))}") + raise ValueError( + f"Unknown {kind} finding ID: {', '.join(sorted(unknown))}" + ) reused = set(ids) & used[kind] if reused: - raise ValueError(f"Conflicting {kind} finding ID: {', '.join(sorted(reused))}") + raise ValueError( + f"Conflicting {kind} finding ID: {', '.join(sorted(reused))}" + ) used[kind].update(ids) matched[kind] = ids if any(matched.values()) and not row["match_reason"].strip(): raise ValueError(f"Reference {reference_id} needs a match reason") - if classification in {"unavailable_evidence", "ambiguous"} and any(matched.values()): + if classification in {"unavailable_evidence", "ambiguous"} and any( + matched.values() + ): raise ValueError(f"Excluded Reference {reference_id} cannot claim coverage") result.append({"reference": reference, "decision": row, "matches": matched}) return result def _metric(caught: int, total: int) -> str: + """Format a binary row-coverage count without inventing an empty rate.""" if total == 0: return "N/A (0 eligible rows)" return f"{caught}/{total} ({caught / total:.1%})" def _md(value: object) -> str: + """Escape source evidence for Markdown without losing line breaks.""" return str(value).replace("\\", "\\\\").replace("|", "\\|").replace("\n", "
") def write_report( - path: Path, manifest: dict[str, Any], rows: list[dict[str, Any]], - audit: list[dict[str, Any]], programmatic: list[dict[str, Any]], + path: Path, + manifest: dict[str, Any], + rows: list[dict[str, Any]], + audit: list[dict[str, Any]], + programmatic: list[dict[str, Any]], ) -> None: + """Write the single human-readable score and row evidence projection.""" if not manifest["complete"]: raise ValueError("Incomplete Evaluation runs cannot produce a final report") findings = {item["finding_id"]: item for item in audit + programmatic} llm = [row for row in rows if row["decision"]["classification"] == "llm_eligible"] - checks = [row for row in rows if row["decision"]["classification"] == "programmatic"] + checks = [ + row for row in rows if row["decision"]["classification"] == "programmatic" + ] covered = [row for row in rows if any(row["matches"].values())] - unavailable = sum(row["decision"]["classification"] == "unavailable_evidence" for row in rows) + audit_caught = sum(bool(row["matches"]["audit"]) for row in llm) + programmatic_caught = sum(bool(row["matches"]["programmatic"]) for row in checks) + unavailable = sum( + row["decision"]["classification"] == "unavailable_evidence" for row in rows + ) ambiguous = sum(row["decision"]["classification"] == "ambiguous" for row in rows) lines = [ - f"# Evaluation run {manifest['run_id']}", "", + f"# Evaluation run {manifest['run_id']}", + "", f"- Model: {manifest['configuration']['model']}", - f"- Workbook-row recall: {_metric(sum(bool(row['matches']['audit']) for row in llm), len(llm))}", - f"- Programmatic coverage: {_metric(sum(bool(row['matches']['programmatic']) for row in checks), len(checks))}", + f"- Workbook-row recall: {_metric(audit_caught, len(llm))}", + f"- Programmatic coverage: {_metric(programmatic_caught, len(checks))}", f"- Combined workbook coverage: {_metric(len(covered), len(rows))}", f"- Estimated run cost: ${manifest['estimated_cost_usd']}", f"- Unavailable evidence: {unavailable}; ambiguous: {ambiguous}", f"- Reported usage: {_md(json.dumps(manifest['usage'], sort_keys=True))}", - "", "This is full-row coverage of this homepage workbook only. A blank finding cell means missed.", - "The cost guardrail is checked between requests; a request can exceed the remaining amount.", - f"Malformed response prompts: {_md(', '.join(manifest['format_failures']) or 'none')}", - "", "## Reference defects", "", + "", + ( + "This is full-row coverage of this homepage workbook only. " + "A blank finding cell means missed." + ), + ( + "The cost guardrail is checked between requests; " + "a request can exceed the remaining amount." + ), + ( + "Malformed response prompts: " + f"{_md(', '.join(manifest['format_failures']) or 'none')}" + ), + "", + "## Reference defects", + "", ] for row in rows: reference = row["reference"] decision = row["decision"] matches = row["matches"] - lines.extend([ - f"### {reference['reference_id']} — {_md(reference['problem'])}", "", - f"- Source: {reference['source_sheet']} row {reference['source_row']} ({reference['page_scope']})", - f"- source_element: {_md(reference['source_element'])}", - f"- Original defect evidence: {_md(json.dumps(reference['raw_evidence'], ensure_ascii=False))}", - f"- Classification: {decision['classification']} — {_md(decision['classification_reason'])}", - f"- Audit findings: {_md('; '.join(matches['audit']) or 'none')}", - f"- Programmatic findings: {_md('; '.join(matches['programmatic']) or 'none')}", - f"- Match reason: {_md(decision['match_reason'] or 'none')}", - f"- Review notes: {_md(decision['review_notes'] or 'none')}", - ]) + evidence = _md(json.dumps(reference["raw_evidence"], ensure_ascii=False)) + lines.extend( + [ + f"### {reference['reference_id']} — {_md(reference['problem'])}", + "", + ( + f"- Source: {reference['source_sheet']} " + f"row {reference['source_row']} ({reference['page_scope']})" + ), + f"- source_element: {_md(reference['source_element'])}", + f"- Original defect evidence: {evidence}", + ( + f"- Classification: {decision['classification']} — " + f"{_md(decision['classification_reason'])}" + ), + f"- Audit findings: {_md('; '.join(matches['audit']) or 'none')}", + ( + "- Programmatic findings: " + f"{_md('; '.join(matches['programmatic']) or 'none')}" + ), + f"- Match reason: {_md(decision['match_reason'] or 'none')}", + f"- Review notes: {_md(decision['review_notes'] or 'none')}", + ] + ) for finding_id in (*matches["audit"], *matches["programmatic"]): finding = findings[finding_id] - lines.append(f"- {finding_id}: {_md(finding['problem'])} @ {_md(finding['location'])}") + lines.append( + f"- {finding_id}: {_md(finding['problem'])} " + f"@ {_md(finding['location'])}" + ) lines.append("") path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index 9c04140..cafd500 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -26,6 +26,7 @@ class ReferenceSet: references: tuple[ReferenceDefect, ...] def to_dict(self) -> dict[str, Any]: + """Return JSON-compatible source evidence.""" return asdict(self) From 5614390a705a41aa222698da789d67a308d33780 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Wed, 23 Sep 2026 18:43:41 -0400 Subject: [PATCH 032/126] docs(evaluation): resolve POC reduction spec --- .scratch/poc-reduction/spec.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.scratch/poc-reduction/spec.md b/.scratch/poc-reduction/spec.md index f6737c9..c251c26 100644 --- a/.scratch/poc-reduction/spec.md +++ b/.scratch/poc-reduction/spec.md @@ -1,6 +1,6 @@ # Reduce the accessibility evaluation proof of concept -Status: ready-for-agent +Status: resolved ## Problem Statement @@ -115,4 +115,8 @@ This specification supersedes earlier evaluation requirements only where they co Inspection found 16 imported Reference defects in the current private artifacts. Their generated eligibility-workbook locations and approved-reference locations matched the imported values, which came from the source `element name` field. This motivates preserving the source label rather than generating a new location. -The agreed size target is fewer than 3,000 added lines for the whole branch change, not merely the evaluation implementation. The existing synthetic CLI workflow and fake-provider seam are the proposed highest practical test seam. Publication marks the work ready for implementation; this task changes the specification only and does not claim the reduction is implemented or verified. +The agreed size target is fewer than 3,000 added lines for the whole branch change, not merely the evaluation implementation. The synthetic CLI workflow and fake-provider seam are the primary test surfaces. + +## Comments + +- Implemented in `7b99d1659e29e9b0b50b6b2abff459625bd25781` and `b32eea12d0dbff2db9f7d9ff003abb955547354b`. Verified with 36 passing tests, evaluator typechecking, lint and format checks, and a 2,940-added-line branch diff against `main`. No paid requests or private artifact staging occurred. From 92859278b1ce6bc140711de140cec2d5b2ac6a3a Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 11:53:50 -0400 Subject: [PATCH 033/126] docs(evaluation): explain final report walkthrough --- README.md | 149 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 775dba0..47c0ab4 100644 --- a/README.md +++ b/README.md @@ -239,59 +239,102 @@ GEMINI_API_KEY=AIza... ## Private Model Evaluation -The private POC measures one fixed Haiku audit against the Home and Global -rows of the authorized Reference workbook. Obtain the workbook from the project -source; the default filename is `Pristine Accessibility Defect Report.xlsx`. -All run evidence stays under ignored `.model-evaluation/runs/`. - -Preview the exact prompt set and estimated paid intent with a local HTML -Benchmark snapshot and the URL it represents: - -```bash -uv run visionaid-evaluate run --html /path/to/home.html \ - --source-url https://pristineai.com/ --max-cost-usd 1.00 -``` - -The CLI copies the HTML, records its SHA-256 and workbook identity, imports -all Home and Global defect rows, runs the existing programmatic checks, and -saves the exact filtered prompts. Preview makes no provider calls even when -credentials are available. It creates a fresh run directory and prints its -path. Use `--workbook PATH` to select another authorized copy of the same -workbook layout. - -For a paid run, repeat the command with `--live`. The CLI shows the fixed -`claude-haiku-4-5-20251001` configuration (16,000 thinking tokens, 24,192 -total output tokens, no temperature), prompt count, pricing, and positive -cost guardrail before asking you to type `yes`. `--approve-live` is an -explicit approval flag for deliberate noninteractive execution. An API key -alone never starts paid requests. The guardrail is checked before each -request; a single request can exceed the remaining amount. Failed requests -are not retried. Responses and reported usage are saved as they arrive, and -incomplete runs cannot be scored. Thinking content is not retained. - -Edit the printed `review.csv` in Excel or a text editor. Classify **every** -row as `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, -and give a short `classification_reason`. Enter full-row matches in -`audit_finding_id` and/or `programmatic_finding_id`; separate multiple IDs -with semicolons and provide a `match_reason`. Leave both ID cells blank for a -miss. Use `review_notes` for partial coverage or location clarification. -`source_element` is the workbook's original `element name`, not an inferred -precise location. Finding IDs and raw evidence are in `audit-findings.json` -and `programmatic-findings.json`. Codex can edit the same CSV on request. - -After human review, generate the Markdown report: - -```bash -uv run visionaid-evaluate report --run-dir .model-evaluation/runs/ -``` - -The report gives Workbook-row recall for LLM-eligible rows, programmatic -coverage for programmatic rows, Combined workbook coverage over all imported -rows, Estimated run cost from API-reported usage and saved pricing, and -source-row explanations. Blank IDs count as misses. A zero eligible -denominator is shown as N/A. This one-homepage result is illustrative and -does not establish broad model equivalence. Historical private runs remain -untouched; the reduced CLI requires fresh runs. +### Get a final report with the supplied Reference workbook + +Run these commands from the repository root. The supplied +`Pristine Accessibility Defect Report.xlsx` is the default Reference workbook; +you do not need to pass `--workbook`. The existing Benchmark snapshot is +`.model-evaluation/snapshots/pristine-homepage/source.html`, and its recorded +source URL is `https://pristineai.com/`. The importer selects its 16 Home and +Global Reference defects. + +1. **Set up live access and choose a limit.** Set `ANTHROPIC_API_KEY` in your + environment or in the repository-root `.env` file (which Git ignores). + Choose a positive USD amount that you approve for this experiment and set + it as `MAX_COST_USD`. The limit is checked before each paid request. A + request already in progress can put the final cost above the limit. + +2. **Start a live Evaluation run.** Run this command in a terminal: + + ```bash + MAX_COST_USD=1.00 # Example; replace with the amount you approve. + uv run visionaid-evaluate run \ + --html .model-evaluation/snapshots/pristine-homepage/source.html \ + --source-url https://pristineai.com/ \ + --max-cost-usd "$MAX_COST_USD" \ + --live + ``` + + The command shows the fixed Haiku model, 16,000 thinking tokens, 24,192 + total output tokens, prompt count, pricing, Benchmark snapshot checksum, + and cost limit. Type `yes` when asked for Live-run approval. The key alone + does not approve spending. The command copies the Benchmark snapshot, + imports the Reference defects, runs programmatic checks, sends the audit + requests, and saves the responses and reported usage. It creates a **new** + directory under `.model-evaluation/runs/`; write down the printed path. + `--approve-live` is available for deliberate noninteractive approval. + +3. **Check completion and review the findings.** Set `RUN_DIR` to the path + printed by the live command. For example, replace the value below with + your actual path: + + ```bash + RUN_DIR='.model-evaluation/runs/' + ``` + + Open `$RUN_DIR/run.json` and confirm `"complete": true`. If it is `false`, + the report command will reject the run. Failed requests are not retried; + start a new live run after you address the failure or cost limit. Do not + reuse the incomplete run. Inspect `$RUN_DIR/audit-findings.json` and + `$RUN_DIR/programmatic-findings.json` for finding IDs, problems, locations, + and raw evidence. `$RUN_DIR/raw-responses.json` has the model responses. + +4. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in + Excel or a text editor. Keep its header, source fields, and one row per + Reference defect. For **all 16 rows**, enter one `classification` value: + `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`. + Enter a short `classification_reason` for every row. `source_element` + repeats the workbook's original `element name`; it is not a precise + inferred location. + + When findings jointly cover a **whole** Reference defect, enter their IDs + in `audit_finding_id` and/or `programmatic_finding_id`. Separate multiple + IDs in one cell with semicolons (`;`). Enter a short `match_reason` when + you enter any ID. Leave both ID cells blank for a miss. Use `review_notes` + to explain partial coverage or location details; notes do not award + partial credit. Do not assign findings to `unavailable_evidence` or + `ambiguous` rows. Save the file as CSV. + +5. **Generate the final report.** Running `report` confirms that human review + is complete: + + ```bash + uv run visionaid-evaluate report --run-dir "$RUN_DIR" + ``` + + Open `$RUN_DIR/report.md`. It contains Workbook-row recall, programmatic + coverage, Combined workbook coverage, Estimated run cost, limitations, + and a source-row explanation for each Reference defect. Reporting rejects + an incomplete run, missing classifications or reasons, and unknown, + wrong-kind, or reused finding IDs. + +The run directory and supplied workbook are excluded from version control. +This one-homepage result is illustrative; it does not establish broad model +equivalence. Historical private runs remain untouched. + +### Check the inputs without a paid request + +Omit `--live` from step 2 to prepare a separate, incomplete Evaluation run. +It copies the Benchmark snapshot, imports the Reference defects, runs +programmatic checks, and saves the exact audit prompts. It does not contact +Haiku or produce Audit findings. It cannot produce a final report. To get a +report, follow steps 1–5 with a **new** live Evaluation run. + +To use a new Benchmark snapshot, save its HTML as a new local file and pass +that file to `--html`. Set `--source-url` to the URL that the HTML represents. +Use the same saved HTML file for input checks and the live run so their input +is identical. The evaluator copies the file and records its checksum; it does +not capture a webpage. ## Running the Web App From bf32ad9e4f6d79b5a84a403308c087e4eead0471 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 15:12:41 -0400 Subject: [PATCH 034/126] docs(evaluation): make walkthrough work after workspace deletion --- README.md | 53 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 47c0ab4..61d2c54 100644 --- a/README.md +++ b/README.md @@ -243,23 +243,38 @@ GEMINI_API_KEY=AIza... Run these commands from the repository root. The supplied `Pristine Accessibility Defect Report.xlsx` is the default Reference workbook; -you do not need to pass `--workbook`. The existing Benchmark snapshot is -`.model-evaluation/snapshots/pristine-homepage/source.html`, and its recorded -source URL is `https://pristineai.com/`. The importer selects its 16 Home and -Global Reference defects. +you do not need to pass `--workbook`. The importer selects its 16 Home and +Global Reference defects. These steps also work after you delete +`.model-evaluation/`. -1. **Set up live access and choose a limit.** Set `ANTHROPIC_API_KEY` in your +1. **Save a Benchmark snapshot.** The evaluator needs a local HTML file. It + does not download the page. Create the ignored directory and save the + homepage HTML there: + + ```bash + mkdir -p .model-evaluation + curl --fail --location --silent --show-error \ + --output .model-evaluation/pristine-home.html https://pristineai.com/ + test -s .model-evaluation/pristine-home.html + ``` + + Open the file and check that it contains the homepage you intend to audit. + `curl` saves the server HTML. If the page builds its content in a browser, + save the rendered HTML to that path instead. Use `https://pristineai.com/` + as the source URL for this file. + +2. **Set up live access and choose a limit.** Set `ANTHROPIC_API_KEY` in your environment or in the repository-root `.env` file (which Git ignores). Choose a positive USD amount that you approve for this experiment and set it as `MAX_COST_USD`. The limit is checked before each paid request. A request already in progress can put the final cost above the limit. -2. **Start a live Evaluation run.** Run this command in a terminal: +3. **Start a live Evaluation run.** Run this command in a terminal: ```bash MAX_COST_USD=1.00 # Example; replace with the amount you approve. uv run visionaid-evaluate run \ - --html .model-evaluation/snapshots/pristine-homepage/source.html \ + --html .model-evaluation/pristine-home.html \ --source-url https://pristineai.com/ \ --max-cost-usd "$MAX_COST_USD" \ --live @@ -274,7 +289,7 @@ Global Reference defects. directory under `.model-evaluation/runs/`; write down the printed path. `--approve-live` is available for deliberate noninteractive approval. -3. **Check completion and review the findings.** Set `RUN_DIR` to the path +4. **Check completion and review the findings.** Set `RUN_DIR` to the path printed by the live command. For example, replace the value below with your actual path: @@ -289,7 +304,7 @@ Global Reference defects. `$RUN_DIR/programmatic-findings.json` for finding IDs, problems, locations, and raw evidence. `$RUN_DIR/raw-responses.json` has the model responses. -4. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in +5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in Excel or a text editor. Keep its header, source fields, and one row per Reference defect. For **all 16 rows**, enter one `classification` value: `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`. @@ -305,7 +320,7 @@ Global Reference defects. partial credit. Do not assign findings to `unavailable_evidence` or `ambiguous` rows. Save the file as CSV. -5. **Generate the final report.** Running `report` confirms that human review +6. **Generate the final report.** Running `report` confirms that human review is complete: ```bash @@ -324,17 +339,13 @@ equivalence. Historical private runs remain untouched. ### Check the inputs without a paid request -Omit `--live` from step 2 to prepare a separate, incomplete Evaluation run. -It copies the Benchmark snapshot, imports the Reference defects, runs -programmatic checks, and saves the exact audit prompts. It does not contact -Haiku or produce Audit findings. It cannot produce a final report. To get a -report, follow steps 1–5 with a **new** live Evaluation run. - -To use a new Benchmark snapshot, save its HTML as a new local file and pass -that file to `--html`. Set `--source-url` to the URL that the HTML represents. -Use the same saved HTML file for input checks and the live run so their input -is identical. The evaluator copies the file and records its checksum; it does -not capture a webpage. +Omit `--live` from step 3 for an input check. This creates a separate, +incomplete Evaluation run with programmatic findings and audit prompts, +but no Haiku requests or Audit findings. It cannot produce a report. +Use a new live run to complete the experiment. + +For another page, pass its saved HTML to `--html` and its URL to +`--source-url`. Keep that HTML file for any input check and live run. ## Running the Web App From 577a8043bc980ab2ae584cb76cc61b27834bc6d5 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 15:28:55 -0400 Subject: [PATCH 035/126] docs(evaluation): show run command without live execution --- README.md | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 61d2c54..2c9f0ed 100644 --- a/README.md +++ b/README.md @@ -241,11 +241,9 @@ GEMINI_API_KEY=AIza... ### Get a final report with the supplied Reference workbook -Run these commands from the repository root. The supplied -`Pristine Accessibility Defect Report.xlsx` is the default Reference workbook; -you do not need to pass `--workbook`. The importer selects its 16 Home and -Global Reference defects. These steps also work after you delete -`.model-evaluation/`. +From the repository root, use the supplied `Pristine Accessibility Defect +Report.xlsx`. It is the default Reference workbook and contains 16 Home and +Global defects. These steps work after you delete `.model-evaluation/`. 1. **Save a Benchmark snapshot.** The evaluator needs a local HTML file. It does not download the page. Create the ignored directory and save the @@ -280,14 +278,12 @@ Global Reference defects. These steps also work after you delete --live ``` - The command shows the fixed Haiku model, 16,000 thinking tokens, 24,192 - total output tokens, prompt count, pricing, Benchmark snapshot checksum, - and cost limit. Type `yes` when asked for Live-run approval. The key alone - does not approve spending. The command copies the Benchmark snapshot, - imports the Reference defects, runs programmatic checks, sends the audit - requests, and saves the responses and reported usage. It creates a **new** - directory under `.model-evaluation/runs/`; write down the printed path. - `--approve-live` is available for deliberate noninteractive approval. + The CLI shows Haiku's 16,000 thinking tokens, 24,192 total output tokens, + prompt count, pricing, snapshot checksum, and limit. Type `yes` to approve + paid requests. A key alone does not approve them. The CLI copies the + Benchmark snapshot, imports Reference defects, runs programmatic checks, + and saves responses and usage in a new `.model-evaluation/runs/` directory. + Keep its printed path. `--approve-live` skips the terminal prompt. 4. **Check completion and review the findings.** Set `RUN_DIR` to the path printed by the live command. For example, replace the value below with @@ -337,15 +333,20 @@ The run directory and supplied workbook are excluded from version control. This one-homepage result is illustrative; it does not establish broad model equivalence. Historical private runs remain untouched. -### Check the inputs without a paid request +### Start a run without `--live` -Omit `--live` from step 3 for an input check. This creates a separate, -incomplete Evaluation run with programmatic findings and audit prompts, -but no Haiku requests or Audit findings. It cannot produce a report. -Use a new live run to complete the experiment. +Save the Benchmark snapshot in step 1, then run this from the repository root: -For another page, pass its saved HTML to `--html` and its URL to -`--source-url`. Keep that HTML file for any input check and live run. +```bash +uv run visionaid-evaluate run --html .model-evaluation/pristine-home.html \ + --source-url https://pristineai.com/ --max-cost-usd 1.00 +``` + +This command needs no API key or approval. It makes a fresh, incomplete +Evaluation run with programmatic findings, audit prompts, and `review.csv`. +It makes no Haiku requests and cannot produce a final report. The CLI still +requires a positive `--max-cost-usd` value; it spends nothing without `--live`. +To finish the evaluation, start a new live run at step 3. ## Running the Web App From 9d63037932ac0db586b04e9558fa6e553309cc19 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 16:01:05 -0400 Subject: [PATCH 036/126] refactor(evaluation)!: name raw and normalized findings explicitly Keep the shared pipeline output name unchanged. New evaluation runs store raw and normalized programmatic findings under distinct names; report reads the normalized file. BREAKING CHANGE: Existing evaluation run directories use unsupported artifact names. --- README.md | 96 ++++++++++++++++-------------------- tests/test_evaluation_cli.py | 10 +++- vision_aid/evaluation/cli.py | 12 +++-- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 2c9f0ed..686e36f 100644 --- a/README.md +++ b/README.md @@ -241,13 +241,11 @@ GEMINI_API_KEY=AIza... ### Get a final report with the supplied Reference workbook -From the repository root, use the supplied `Pristine Accessibility Defect -Report.xlsx`. It is the default Reference workbook and contains 16 Home and -Global defects. These steps work after you delete `.model-evaluation/`. +From the repository root, use the supplied `Pristine Accessibility Defect Report.xlsx` +(16 Home and Global defects). These steps work after deleting `.model-evaluation/`. -1. **Save a Benchmark snapshot.** The evaluator needs a local HTML file. It - does not download the page. Create the ignored directory and save the - homepage HTML there: +1. **Save a Benchmark snapshot.** The evaluator does not download the page. + Create the ignored directory and save the homepage HTML there: ```bash mkdir -p .model-evaluation @@ -256,16 +254,14 @@ Global defects. These steps work after you delete `.model-evaluation/`. test -s .model-evaluation/pristine-home.html ``` - Open the file and check that it contains the homepage you intend to audit. - `curl` saves the server HTML. If the page builds its content in a browser, - save the rendered HTML to that path instead. Use `https://pristineai.com/` - as the source URL for this file. + Check that the file contains the intended homepage. `curl` saves server HTML; + if the browser builds the content, save rendered HTML there instead. Use + `https://pristineai.com/` as its source URL. 2. **Set up live access and choose a limit.** Set `ANTHROPIC_API_KEY` in your - environment or in the repository-root `.env` file (which Git ignores). - Choose a positive USD amount that you approve for this experiment and set - it as `MAX_COST_USD`. The limit is checked before each paid request. A - request already in progress can put the final cost above the limit. + environment or the ignored repository-root `.env`. Set `MAX_COST_USD` to a + positive amount you approve. The limit is checked before each paid request; + a request in progress can put the final cost above it. 3. **Start a live Evaluation run.** Run this command in a terminal: @@ -278,60 +274,51 @@ Global defects. These steps work after you delete `.model-evaluation/`. --live ``` - The CLI shows Haiku's 16,000 thinking tokens, 24,192 total output tokens, + The CLI shows Haiku's 16,000 thinking tokens, 24,192 output token cap, prompt count, pricing, snapshot checksum, and limit. Type `yes` to approve - paid requests. A key alone does not approve them. The CLI copies the - Benchmark snapshot, imports Reference defects, runs programmatic checks, - and saves responses and usage in a new `.model-evaluation/runs/` directory. - Keep its printed path. `--approve-live` skips the terminal prompt. + paid requests; `--approve-live` skips the prompt. It copies the snapshot, + imports Reference defects, runs checks, and saves responses and usage. + Keep the printed run directory path. -4. **Check completion and review the findings.** Set `RUN_DIR` to the path - printed by the live command. For example, replace the value below with - your actual path: +4. **Check completion and review the findings.** Use the printed run directory: ```bash RUN_DIR='.model-evaluation/runs/' ``` - Open `$RUN_DIR/run.json` and confirm `"complete": true`. If it is `false`, - the report command will reject the run. Failed requests are not retried; - start a new live run after you address the failure or cost limit. Do not - reuse the incomplete run. Inspect `$RUN_DIR/audit-findings.json` and - `$RUN_DIR/programmatic-findings.json` for finding IDs, problems, locations, - and raw evidence. `$RUN_DIR/raw-responses.json` has the model responses. + Confirm `"complete": true` in `$RUN_DIR/run.json`; otherwise, address the + failure or cost limit and start a new live run. Failed requests are not + retried. Use `$RUN_DIR/audit-findings.json` and + `$RUN_DIR/normalized-programmatic-findings.json` for finding IDs and evidence. + `$RUN_DIR/raw-programmatic-findings.json` has the original checker output; + `$RUN_DIR/raw-responses.json` has model responses. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in - Excel or a text editor. Keep its header, source fields, and one row per - Reference defect. For **all 16 rows**, enter one `classification` value: + Excel or a text editor. Keep its header, source fields, and all 16 rows. + Enter one `classification` value per Reference defect: `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`. - Enter a short `classification_reason` for every row. `source_element` - repeats the workbook's original `element name`; it is not a precise - inferred location. + Enter a short `classification_reason` for every row. `source_element` is + the workbook's original `element name`, not an inferred location. When findings jointly cover a **whole** Reference defect, enter their IDs - in `audit_finding_id` and/or `programmatic_finding_id`. Separate multiple - IDs in one cell with semicolons (`;`). Enter a short `match_reason` when - you enter any ID. Leave both ID cells blank for a miss. Use `review_notes` - to explain partial coverage or location details; notes do not award - partial credit. Do not assign findings to `unavailable_evidence` or - `ambiguous` rows. Save the file as CSV. + in `audit_finding_id` and/or `programmatic_finding_id`, separating multiple + IDs with semicolons (`;`). Enter a `match_reason` for any ID. Leave both ID + cells blank for a miss. Use `review_notes` for partial coverage or location + details; notes do not award partial credit. Do not assign findings to + `unavailable_evidence` or `ambiguous` rows. Save as CSV. -6. **Generate the final report.** Running `report` confirms that human review - is complete: +6. **Generate the final report.** Run `report` after human review: ```bash uv run visionaid-evaluate report --run-dir "$RUN_DIR" ``` - Open `$RUN_DIR/report.md`. It contains Workbook-row recall, programmatic - coverage, Combined workbook coverage, Estimated run cost, limitations, - and a source-row explanation for each Reference defect. Reporting rejects - an incomplete run, missing classifications or reasons, and unknown, - wrong-kind, or reused finding IDs. + Open `$RUN_DIR/report.md` for Workbook-row recall, programmatic coverage, + Combined workbook coverage, Estimated run cost, limitations, and source-row + explanations. Reporting rejects incomplete runs, missing decisions, and + invalid or reused finding IDs. -The run directory and supplied workbook are excluded from version control. -This one-homepage result is illustrative; it does not establish broad model -equivalence. Historical private runs remain untouched. +The run directory and workbook are excluded from version control. ### Start a run without `--live` @@ -342,11 +329,12 @@ uv run visionaid-evaluate run --html .model-evaluation/pristine-home.html \ --source-url https://pristineai.com/ --max-cost-usd 1.00 ``` -This command needs no API key or approval. It makes a fresh, incomplete -Evaluation run with programmatic findings, audit prompts, and `review.csv`. -It makes no Haiku requests and cannot produce a final report. The CLI still -requires a positive `--max-cost-usd` value; it spends nothing without `--live`. -To finish the evaluation, start a new live run at step 3. +This needs no API key or approval. It creates an incomplete run with +`raw-programmatic-findings.json`, `normalized-programmatic-findings.json`, +audit prompts, and `review.csv`. Use the normalized file's `finding_id` values +for programmatic matches. It makes no Haiku requests and cannot produce a +final report. `--max-cost-usd` must be positive but is not spent without +`--live`. Start a new live run at step 3 to finish the evaluation. ## Running the Web App diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 0e150f5..d9bcbe8 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -143,6 +143,11 @@ def test_preview_preserves_reference_rows_and_never_calls_provider( assert "claude-haiku-4-5-20251001" in output assert "16000" in output and "24192" in output and "$0.01" in output assert (run / "snapshot.html").read_bytes() == html.read_bytes() + raw = json.loads((run / "raw-programmatic-findings.json").read_text()) + normalized = json.loads((run / "normalized-programmatic-findings.json").read_text()) + assert raw and [finding["raw_source"] for finding in normalized] == raw + assert not (run / "programmatic_findings.json").exists() + assert not (run / "programmatic-findings.json").exists() manifest = json.loads((run / "run.json").read_text()) assert manifest["snapshot_sha256"] == hashlib.sha256(html.read_bytes()).hexdigest() rows = read_review(run / "review.csv") @@ -213,7 +218,8 @@ def client(**options: object) -> FakeClient: json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" ) findings = json.loads((run / "audit-findings.json").read_text()) - programmatic = json.loads((run / "programmatic-findings.json").read_text()) + programmatic_file = run / "normalized-programmatic-findings.json" + programmatic = json.loads(programmatic_file.read_text()) assert findings and programmatic review = run / "review.csv" rows = read_review(review) @@ -268,7 +274,7 @@ def call(self, prompt: str) -> dict: item["finding_id"] for item in json.loads((run / "audit-findings.json").read_text()) ] - programmatic_id = json.loads((run / "programmatic-findings.json").read_text())[0][ + programmatic_id = json.loads((run / "normalized-programmatic-findings.json").read_text())[0][ "finding_id" ] rows = read_review(review) diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 1c64cd5..313fafe 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -102,13 +102,18 @@ def _run(args: argparse.Namespace) -> int: request_config=REQUEST_CONFIG, ) (run / "manifest.json").unlink() - raw_programmatic = json.loads((run / "programmatic_findings.json").read_text()) + raw_path = run / "programmatic_findings.json" + raw_programmatic = json.loads(raw_path.read_text()) + raw_path.rename(run / "raw-programmatic-findings.json") findings = normalize_programmatic_findings( raw_programmatic, run_id=run.name, page_url=args.source_url, ) - write_json(run / "programmatic-findings.json", [asdict(item) for item in findings]) + write_json( + run / "normalized-programmatic-findings.json", + [asdict(item) for item in findings], + ) create_review(run / "review.csv", list(references.to_dict()["references"])) schedule = PriceSchedule.load(run / "pricing.json") manifest = { @@ -189,7 +194,8 @@ def _report(args: argparse.Namespace) -> int: if reference_set["workbook_sha256"] != manifest["workbook_sha256"]: raise ValueError("Reference workbook identity differs from the run") audit = json.loads((run / "audit-findings.json").read_text()) - programmatic = json.loads((run / "programmatic-findings.json").read_text()) + programmatic_file = run / "normalized-programmatic-findings.json" + programmatic = json.loads(programmatic_file.read_text()) for kind, collection in (("audit", audit), ("programmatic", programmatic)): if any( item["run_id"] != run.name From 2fb478cff0b3e0c70b76c0dd5af8bacbe617731f Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 16:02:35 -0400 Subject: [PATCH 037/126] docs(evaluation): retain single-page limitation after rename Keep the walkthrough limitation raised in review and apply the test formatter. --- README.md | 2 +- tests/test_evaluation_cli.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 686e36f..fed220f 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,6 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report prompt count, pricing, snapshot checksum, and limit. Type `yes` to approve paid requests; `--approve-live` skips the prompt. It copies the snapshot, imports Reference defects, runs checks, and saves responses and usage. - Keep the printed run directory path. 4. **Check completion and review the findings.** Use the printed run directory: @@ -319,6 +318,7 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report invalid or reused finding IDs. The run directory and workbook are excluded from version control. +This one-homepage result does not establish broad model equivalence. ### Start a run without `--live` diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index d9bcbe8..0a02a3c 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -274,9 +274,9 @@ def call(self, prompt: str) -> dict: item["finding_id"] for item in json.loads((run / "audit-findings.json").read_text()) ] - programmatic_id = json.loads((run / "normalized-programmatic-findings.json").read_text())[0][ - "finding_id" - ] + programmatic_id = json.loads( + (run / "normalized-programmatic-findings.json").read_text() + )[0]["finding_id"] rows = read_review(review) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) From aad520401189a70564225d3fe15dcbf0fe62d4a8 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 24 Sep 2026 16:16:27 -0400 Subject: [PATCH 038/126] docs(evaluation): make filling in review.csv more clear --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index fed220f..504a904 100644 --- a/README.md +++ b/README.md @@ -293,18 +293,8 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report `$RUN_DIR/raw-responses.json` has model responses. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in - Excel or a text editor. Keep its header, source fields, and all 16 rows. - Enter one `classification` value per Reference defect: - `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`. - Enter a short `classification_reason` for every row. `source_element` is - the workbook's original `element name`, not an inferred location. - - When findings jointly cover a **whole** Reference defect, enter their IDs - in `audit_finding_id` and/or `programmatic_finding_id`, separating multiple - IDs with semicolons (`;`). Enter a `match_reason` for any ID. Leave both ID - cells blank for a miss. Use `review_notes` for partial coverage or location - details; notes do not award partial credit. Do not assign findings to - `unavailable_evidence` or `ambiguous` rows. Save as CSV. + Excel or a CSV-aware text editor. Review every Reference defect and fill in + the decision columns using the guide below. Save the file as CSV. 6. **Generate the final report.** Run `report` after human review: @@ -317,6 +307,47 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report explanations. Reporting rejects incomplete runs, missing decisions, and invalid or reused finding IDs. +### Fill in `review.csv` + +The CSV has one row for each of the 16 Home or Global Reference defects. Keep +the header, all rows, and their `reference_id` values. The columns from +`reference_id` through `comment` identify the workbook row and preserve its +original evidence. Use them to understand the defect, but do not edit them. +In particular, `source_sheet` and `source_row` point back to the workbook, +`page_scope` says Home or Global, and `source_element` copies the workbook's +`element name`. It is not an inferred location on the page. + +For **every row**, fill in `classification` and `classification_reason`: + +| `classification` value | Use it when | +| --- | --- | +| `llm_eligible` | The Benchmark snapshot contains enough evidence to judge the defect, and the issue calls for the AI audit's judgment. | +| `programmatic` | A deterministic check can establish the defect from the available evidence. Classification alone does not mean the checker found it. | +| `unavailable_evidence` | The snapshot lacks evidence needed to verify the defect, such as behavior that requires interaction or a browser or assistive technology observation. | +| `ambiguous` | The Reference defect is too unclear to decide fairly whether a finding covers it. | + +Base the classification on the Reference defect and available evidence, not on +whether this run happened to produce a matching finding. In +`classification_reason`, write a short explanation of that choice, including +what evidence is missing or unclear when applicable. These two cells cannot be +blank when you generate the report. + +Then complete the remaining decision columns for that row: + +| Column | What to enter | +| --- | --- | +| `audit_finding_id` | Copy `finding_id` values from `$RUN_DIR/audit-findings.json` only when the selected AI Audit findings **jointly cover the whole Reference defect**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | +| `programmatic_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-programmatic-findings.json` only when the selected Programmatic findings **jointly cover the whole Reference defect**. Use semicolons for multiple IDs. Otherwise leave blank. | +| `match_reason` | If either finding-ID cell has an ID, briefly explain how those findings cover the full defect, including the relevant element or behavior. If both ID cells are blank, this can be blank. | +| `review_notes` | Optional context, such as a more precise location or a finding that covers only part of the defect. Notes do not award coverage. | + +You may use findings from both files for one row. Each finding ID must exist in +the named file and can be assigned to only one Reference defect. Leave both ID +cells blank when no findings fully cover the row; that records a miss, even if +you describe partial evidence in `review_notes`. For `unavailable_evidence` and +`ambiguous` rows, leave both ID cells blank. The report command checks these +rules and requires a `match_reason` whenever you enter an ID. + The run directory and workbook are excluded from version control. This one-homepage result does not establish broad model equivalence. From 28c8a869c1b9926f899f10e3ff02f8741b62c17b Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 17:47:48 -0400 Subject: [PATCH 039/126] feat(skills): add link to matt pocock skills for claude --- CLAUDE.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 220212a..5a9d342 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,6 +5,7 @@ This project builds LLM-powered tools that analyze websites for WCAG accessibility issues and generate structured remediation reports. It is a Computing for Good course project at Georgia Tech (OMSCS) partnered with the Vision Aid Digital Accessibility Testing Team. The system has three planned stages: + 1. Automated webpage accessibility analysis → structured CSV reports 2. Code remediation suggestions based on those reports 3. (Stretch) Chrome extension for on-the-fly accessibility fixes @@ -116,3 +117,17 @@ Docker image → GHCR (`ghcr.io/c4g/va-dat`) → Coolify at `https://va-dat.c4g. - HTML test files can be enormous (500K+ tokens) — do not read them fully into context. - Some files may have non-UTF-8 encoding; always use `errors='replace'`. - Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-5`, `api_server.py` uses `claude-haiku-4-5-20251001`. + +## Agent skills + +### Issue tracker + +Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`. + +### Domain docs + +This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`. From 9c014de34222aefa2940cee7fe9c36554baefd32 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 18:26:34 -0400 Subject: [PATCH 040/126] docs(evaluation): explain homepage run JSON files --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 504a904..77ff4ec 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,41 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report explanations. Reporting rejects incomplete runs, missing decisions, and invalid or reused finding IDs. +### Evaluation run files + +The `run` command saves these files under `$RUN_DIR`. Without `--live`, it +prepares the prompts but makes no model requests. + +| File | Role | +| --- | --- | +| `references.json` | The imported Home and Global Reference defects, their original workbook evidence, and stable IDs. | +| `payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | +| `payloads/cl02_payload.json` | Extracted forms, fields, and labels. | +| `payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | +| `prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | +| `raw-programmatic-findings.json` | Original output from the deterministic accessibility checks. | +| `normalized-programmatic-findings.json` | Those checker results converted to findings with `finding_id` values for `review.csv`. | +| `raw-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | +| `audit-findings.json` | Parsed AI Audit findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-responses.json` without producing a finding here. | +| `run.json` | The run's configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | +| `pricing.json` | A copy of the versioned model rates used to estimate this run's cost from reported usage. | + +The payload files show extracted page data before filtering. The prompt files +show what the model actually received after filtering and slicing. To review a +Reference defect, compare `references.json` with the two normalized findings +files and inspect `snapshot.html` for the full Benchmark evidence. Check +`raw-responses.json` when an AI finding is missing or malformed, and check the +prompt file to see whether the model received the relevant evidence. Record +decisions in `review.csv`; the `report` command reads that CSV and the run +evidence to write `report.md`. The pipeline briefly writes `manifest.json`, but +the Evaluation command removes it and keeps the relevant metadata in `run.json`. + +Earlier private experiments may also contain `preflight.json` with advance +token counts and a spending plan, `format-diagnostics.json` with an offline +inspection of malformed replies, and `review-validation.json` with checks on +the completed review. The standard `visionaid-evaluate run` command does not +create those files or perform those extra checks. + ### Fill in `review.csv` The CSV has one row for each of the 16 Home or Global Reference defects. Keep From d4a87cf5e36e401da2898863c501b934c972fdb3 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 18:43:25 -0400 Subject: [PATCH 041/126] refactor(evaluation)!: name raw and normalized audit files explicitly Rename raw-responses.json to raw-audit-responses.json and audit-findings.json to normalized-audit-findings.json, matching the raw-/normalized- programmatic findings files. The raw file keeps "responses" because it records failed and malformed replies that contain no findings. BREAKING CHANGE: Existing evaluation run directories use unsupported artifact names. Co-Authored-By: Claude Opus 5.5 (1M context) --- .scratch/rename-audit-artifacts/spec.md | 69 +++++++++++++++++++++++++ README.md | 23 +++++---- tests/test_evaluation_cli.py | 9 ++-- vision_aid/evaluation/audit.py | 4 +- vision_aid/evaluation/cli.py | 2 +- 5 files changed, 90 insertions(+), 17 deletions(-) create mode 100644 .scratch/rename-audit-artifacts/spec.md diff --git a/.scratch/rename-audit-artifacts/spec.md b/.scratch/rename-audit-artifacts/spec.md new file mode 100644 index 0000000..0ef9b5c --- /dev/null +++ b/.scratch/rename-audit-artifacts/spec.md @@ -0,0 +1,69 @@ +# Name raw and normalized audit artifacts explicitly + +Status: ready-for-agent + +## Problem Statement + +An Evaluation run directory stores Programmatic findings under two explicit names, `raw-programmatic-findings.json` and `normalized-programmatic-findings.json`, but stores the AI audit's evidence as `raw-responses.json` and `audit-findings.json`. The two families do not read as parallel: nothing in the audit names says which file is the raw model evidence and which is the normalized set a reviewer copies `finding_id` values from. An evaluator filling in `review.csv` has to consult the README to know which file to open. + +## Solution + +Rename the audit artifacts so that both finding kinds follow the same `raw-…` / `normalized-…` convention, using the glossary term **Audit finding**: + +- `raw-responses.json` becomes `raw-audit-responses.json` +- `audit-findings.json` becomes `normalized-audit-findings.json` + +The raw file keeps the noun "responses" because it holds one entry per prompt, including failed requests, malformed replies, and usage, which are not findings. The normalized file keeps "findings" because it holds Audit findings with `finding_id` values. + +This is a clean break, matching the earlier programmatic rename: new Evaluation runs write only the new names, and `report` reads only the new names. Existing local run directories are discarded rather than migrated. + +## User Stories + +1. As an evaluator, I want the normalized Audit findings file to be named `normalized-audit-findings.json`, so that I can tell at a glance it is the file to copy `audit_finding_id` values from. +2. As an evaluator, I want the raw model evidence file to be named `raw-audit-responses.json`, so that I can tell it holds the unprocessed model replies for the AI audit. +3. As an evaluator, I want the audit and programmatic artifacts to share a `raw-…` / `normalized-…` naming pattern, so that I can find the matching file for either finding kind without reading documentation. +4. As an evaluator, I want the raw audit file to still be called "responses", so that I am not misled into expecting findings for requests that failed or returned malformed output. +5. As an evaluator, I want `report` to read Audit findings from `normalized-audit-findings.json`, so that scoring uses the same file I reviewed. +6. As an evaluator, I want a live Evaluation run to stop writing `raw-responses.json` and `audit-findings.json`, so that a run directory never holds two competing copies of the same evidence. +7. As an evaluator, I want a run stopped by a request failure or the cost guardrail to still record its responses in `raw-audit-responses.json`, so that partial runs remain traceable. +8. As an evaluator, I want `raw-audit-responses.json` and `normalized-audit-findings.json` to be rewritten after every prompt, as the current files are, so that an interrupted run keeps the evidence gathered so far. +9. As an evaluator, I want the README's review instructions to name the new files, so that the steps I follow match what is on disk. +10. As an evaluator, I want the README's run-directory file table to list the raw and normalized files for each finding kind next to each other, so that the parallel structure is visible. +11. As an evaluator, I want the README's troubleshooting guidance to point to `raw-audit-responses.json` when an Audit finding is missing or malformed, so that I look in the right place. +12. As a maintainer, I want the `review.csv` column `audit_finding_id` to stay unchanged, so that the column keeps matching the glossary term Audit finding. +13. As a maintainer, I want the change marked as breaking in its commit, so that anyone holding an old run directory knows `report` will no longer read it. +14. As a maintainer, I want no fallback for the old filenames, so that the evaluator does not carry compatibility code for throwaway local output. +15. As a maintainer, I want the tests to fail if the old filenames are ever written again, so that the rename cannot silently regress. + +## Implementation Decisions + +- The audit execution step that iterates over prompts writes its per-prompt evidence to `raw-audit-responses.json` and its accumulated Audit findings to `normalized-audit-findings.json`. File contents and shapes are unchanged; only the names change. +- The `report` command loads Audit findings from `normalized-audit-findings.json`. Its existing failure when audit evidence is absent (for example, after a preview-only run) is unchanged. +- No compatibility layer: old names are neither read nor written. +- The `review.csv` column `audit_finding_id`, the `run.json` manifest fields (including `format_failures`), and the programmatic artifact names are unchanged. +- `CONTEXT.md` is unchanged. Filenames are an implementation detail, and "Audit finding" remains the correct term. No ADR is needed, since the rename is cheap to reverse. +- README updates cover the review-workflow step that names the finding files, the run-directory file table, the troubleshooting paragraph, the `audit_finding_id` column description, and the list of run artifacts. +- The commit uses the `refactor(evaluation)!:` prefix with a `BREAKING CHANGE:` footer stating that existing Evaluation run directories use unsupported artifact names, following the precedent of the earlier programmatic-findings rename. +- Existing local run directories under the gitignored `.model-evaluation/runs/` are deleted rather than migrated. The rest of `.model-evaluation/` (snapshots, plans) is kept. + +## Testing Decisions + +- Tests exercise external behavior only, through the evaluation CLI's `main` entry point with the audit request client replaced by a fake. That is the single seam, and it already exists. Tests assert on which files exist in the run directory and on `report`'s outcome, not on internal helpers. +- Live-run test: after a live run with a fake client, `raw-audit-responses.json` and `normalized-audit-findings.json` exist; `raw-responses.json` and `audit-findings.json` do not; and `report` succeeds when `review.csv` references `finding_id` values read from `normalized-audit-findings.json`. +- Failure or guardrail test: after a run that stops on the first prompt, `raw-audit-responses.json` holds exactly one response, and `report` still refuses the incomplete run. +- Preview test: unchanged. `report` still fails when no audit artifacts exist. +- Prior art: the existing CLI tests already assert that the old programmatic filenames are absent after a preview run. Mirror that pattern for the old audit filenames. +- No new unit tests at the audit-module level. + +## Out of Scope + +- Renaming the Audit finding concept (for example, to "LLM finding"), the `audit_finding_id` review column, or the `llm_eligible` classification. +- Changing the contents or schema of either audit artifact. +- Migrating or reading old run directories. +- Renaming prompt, payload, snapshot, reference, pricing, or manifest files. +- Any change to the shared pipeline output used by the web server or `run_pipeline.py`. + +## Further Notes + +- Considered and rejected: `raw-llm-responses.json` / `normalized-llm-responses.json`. The `llm` prefix would give one concept a third name alongside "Audit finding" and "AI Audit". The normalized file holds findings, not responses. +- Considered and rejected: `raw-audit-findings.json`. Failed and malformed replies recorded in the raw file contain no findings. diff --git a/README.md b/README.md index 77ff4ec..e5a6dc4 100644 --- a/README.md +++ b/README.md @@ -287,10 +287,10 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report Confirm `"complete": true` in `$RUN_DIR/run.json`; otherwise, address the failure or cost limit and start a new live run. Failed requests are not - retried. Use `$RUN_DIR/audit-findings.json` and + retried. Use `$RUN_DIR/normalized-audit-findings.json` and `$RUN_DIR/normalized-programmatic-findings.json` for finding IDs and evidence. - `$RUN_DIR/raw-programmatic-findings.json` has the original checker output; - `$RUN_DIR/raw-responses.json` has model responses. + `$RUN_DIR/raw-audit-responses.json` has model responses; + `$RUN_DIR/raw-programmatic-findings.json` has the original checker output. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in Excel or a CSV-aware text editor. Review every Reference defect and fill in @@ -321,8 +321,8 @@ prepares the prompts but makes no model requests. | `prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | | `raw-programmatic-findings.json` | Original output from the deterministic accessibility checks. | | `normalized-programmatic-findings.json` | Those checker results converted to findings with `finding_id` values for `review.csv`. | -| `raw-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | -| `audit-findings.json` | Parsed AI Audit findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-responses.json` without producing a finding here. | +| `raw-audit-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | +| `normalized-audit-findings.json` | Parsed AI Audit findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-audit-responses.json` without producing a finding here. | | `run.json` | The run's configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | | `pricing.json` | A copy of the versioned model rates used to estimate this run's cost from reported usage. | @@ -330,11 +330,12 @@ The payload files show extracted page data before filtering. The prompt files show what the model actually received after filtering and slicing. To review a Reference defect, compare `references.json` with the two normalized findings files and inspect `snapshot.html` for the full Benchmark evidence. Check -`raw-responses.json` when an AI finding is missing or malformed, and check the -prompt file to see whether the model received the relevant evidence. Record -decisions in `review.csv`; the `report` command reads that CSV and the run -evidence to write `report.md`. The pipeline briefly writes `manifest.json`, but -the Evaluation command removes it and keeps the relevant metadata in `run.json`. +`raw-audit-responses.json` when an AI finding is missing or malformed, and +check the prompt file to see whether the model received the relevant evidence. +Record decisions in `review.csv`; the `report` command reads that CSV and the +run evidence to write `report.md`. The pipeline briefly writes `manifest.json`, +but the Evaluation command removes it and keeps the relevant metadata in +`run.json`. Earlier private experiments may also contain `preflight.json` with advance token counts and a spending plan, `format-diagnostics.json` with an offline @@ -371,7 +372,7 @@ Then complete the remaining decision columns for that row: | Column | What to enter | | --- | --- | -| `audit_finding_id` | Copy `finding_id` values from `$RUN_DIR/audit-findings.json` only when the selected AI Audit findings **jointly cover the whole Reference defect**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | +| `audit_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-audit-findings.json` only when the selected AI Audit findings **jointly cover the whole Reference defect**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | | `programmatic_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-programmatic-findings.json` only when the selected Programmatic findings **jointly cover the whole Reference defect**. Use semicolons for multiple IDs. Otherwise leave blank. | | `match_reason` | If either finding-ID cell has an ID, briefly explain how those findings cover the full defect, including the relevant element or behavior. If both ID cells are blank, this can be blank. | | `review_notes` | Optional context, such as a more precise location or a finding that covers only part of the defect. Notes do not award coverage. | diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 0a02a3c..ef4c4f5 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -217,7 +217,10 @@ def client(**options: object) -> FakeClient: assert ( json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" ) - findings = json.loads((run / "audit-findings.json").read_text()) + assert (run / "raw-audit-responses.json").exists() + assert not (run / "raw-responses.json").exists() + assert not (run / "audit-findings.json").exists() + findings = json.loads((run / "normalized-audit-findings.json").read_text()) programmatic_file = run / "normalized-programmatic-findings.json" programmatic = json.loads(programmatic_file.read_text()) assert findings and programmatic @@ -272,7 +275,7 @@ def call(self, prompt: str) -> dict: review = run / "review.csv" audit_ids = [ item["finding_id"] - for item in json.loads((run / "audit-findings.json").read_text()) + for item in json.loads((run / "normalized-audit-findings.json").read_text()) ] programmatic_id = json.loads( (run / "normalized-programmatic-findings.json").read_text() @@ -378,6 +381,6 @@ def call(self, prompt: str) -> dict: assert manifest["complete"] is False assert manifest["usage"]["input_tokens"] == 100000 assert float(manifest["estimated_cost_usd"]) > 0 - assert len(json.loads((run / "raw-responses.json").read_text())) == 1 + assert len(json.loads((run / "raw-audit-responses.json").read_text())) == 1 with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/audit.py index 50659d4..d37d2f8 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/audit.py @@ -114,8 +114,8 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: incomplete_reason=reason, format_failures=format_failures, ) - write_json(run / "raw-responses.json", responses) - write_json(run / "audit-findings.json", findings) + write_json(run / "raw-audit-responses.json", responses) + write_json(run / "normalized-audit-findings.json", findings) write_json(run / "run.json", manifest) if not complete: break diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 313fafe..8327911 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -193,7 +193,7 @@ def _report(args: argparse.Namespace) -> int: reference_set = json.loads((run / "references.json").read_text()) if reference_set["workbook_sha256"] != manifest["workbook_sha256"]: raise ValueError("Reference workbook identity differs from the run") - audit = json.loads((run / "audit-findings.json").read_text()) + audit = json.loads((run / "normalized-audit-findings.json").read_text()) programmatic_file = run / "normalized-programmatic-findings.json" programmatic = json.loads(programmatic_file.read_text()) for kind, collection in (("audit", audit), ("programmatic", programmatic)): From 14c07a795b3b618def05d31efbe885e2dd8d6630 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 18:43:29 -0400 Subject: [PATCH 042/126] docs(scratch): resolve rename-audit-artifacts spec Co-Authored-By: Claude Opus 5.5 (1M context) --- .scratch/rename-audit-artifacts/spec.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.scratch/rename-audit-artifacts/spec.md b/.scratch/rename-audit-artifacts/spec.md index 0ef9b5c..ea6efe2 100644 --- a/.scratch/rename-audit-artifacts/spec.md +++ b/.scratch/rename-audit-artifacts/spec.md @@ -1,6 +1,6 @@ # Name raw and normalized audit artifacts explicitly -Status: ready-for-agent +Status: resolved ## Problem Statement @@ -67,3 +67,7 @@ This is a clean break, matching the earlier programmatic rename: new Evaluation - Considered and rejected: `raw-llm-responses.json` / `normalized-llm-responses.json`. The `llm` prefix would give one concept a third name alongside "Audit finding" and "AI Audit". The normalized file holds findings, not responses. - Considered and rejected: `raw-audit-findings.json`. Failed and malformed replies recorded in the raw file contain no findings. + +## Comments + +Implemented in d4a87cf5. Verified with the evaluation CLI tests at the `main` seam: a live run writes `raw-audit-responses.json` and `normalized-audit-findings.json` but not the old names, `report` scores from the normalized file, and a stopped run records one response. Full suite: 36 passed. The two-axis review found no spec gaps. Existing local run directories were deleted. From 25cd709dc3a71a50bf0ed6d84a7055602349e089 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 19:16:31 -0400 Subject: [PATCH 043/126] refactor(evaluation): rename pricing.v2.json to pricing.json The schedule version lives inside the file, and run folders already store their copy as pricing.json. Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 2 +- vision_aid/evaluation/{pricing.v2.json => pricing.json} | 0 vision_aid/evaluation/pricing.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename vision_aid/evaluation/{pricing.v2.json => pricing.json} (100%) diff --git a/pyproject.toml b/pyproject.toml index 6de5cd1..3d7ca33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.v2.json"] +"vision_aid.evaluation" = ["pricing.json"] "processing_scripts.llm" = ["*.txt"] diff --git a/vision_aid/evaluation/pricing.v2.json b/vision_aid/evaluation/pricing.json similarity index 100% rename from vision_aid/evaluation/pricing.v2.json rename to vision_aid/evaluation/pricing.json diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index f3ba799..4083125 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any -DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.v2.json") +DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.json") MILLION = Decimal(1_000_000) From 30ca95fdd73ee4eac64ae90e0e0634072cebac5a Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sun, 27 Sep 2026 19:57:20 -0400 Subject: [PATCH 044/126] fix(evaluation): render original defect evidence verbatim in report.md Raw HTML such as
in workbook evidence was swallowed by Markdown preview, and the evidence was one unreadable JSON line. Each Reference defect now ends with a collapsible
block holding the evidence as pretty-printed JSON in a fence sized past any backtick run in the content. Inline fields escape < and > so markup in reasons and findings stays visible. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_evaluation_cli.py | 57 +++++++++++++++++++++++++++++++++ vision_aid/evaluation/review.py | 27 +++++++++++++--- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index ef4c4f5..78edd07 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -250,6 +250,63 @@ def client(**options: object) -> FakeClient: assert "Vague link" in report and "First link" in report +def test_report_shows_evidence_verbatim_and_escapes_inline_html( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fenced evidence stays verbatim and inline fields escape raw HTML.""" + workbook, html = inputs(tmp_path) + book = openpyxl.load_workbook(workbook) + actual = 'Use
tag.\nSay "Main" ```here```' + book.active["G2"] = actual + book.save(workbook) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + def call(self, prompt: str) -> dict: + """Return one normalized finding for applicable prompts.""" + return { + "success": True, + "response": json.dumps([{"problem": "Vague link"}]), + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: FakeClient()) + assert main(args(workbook, html, "--live", "--approve-live")) == 0 + run = directory(tmp_path) + review = run / "review.csv" + rows = read_review(review) + for row in rows: + row.update(classification="ambiguous", classification_reason="Unclear") + rows[0]["classification_reason"] = 'An needs judgment' + write_review(review, rows) + assert main(["report", "--run-dir", str(run)]) == 0 + report = (run / "report.md").read_text() + evidence = ( + "
Original defect evidence\n\n" + "````json\n" + "{\n" + ' "Browser Combination": "Chrome",\n' + ' "Issue Title": "Unclear link",\n' + ' "Page name": "Home",\n' + ' "Recommendation for Fix": "Rename",\n' + ' "Sr. #": "1",\n' + ' "Steps to Reproduce": "Open page",\n' + ' "Type of Change": "Code",\n' + ' "WCAG Sc": "2.4.4",\n' + ' "actual result": "Use
tag.\\nSay \\"Main\\" ```here```",\n' + ' "comment": "",\n' + ' "element name": "First link",\n' + ' "expected result": "Descriptive text"\n' + "}\n" + "````\n\n" + "
\n" + ) + assert evidence in report + assert "Original defect evidence:" not in report + assert 'An \\ needs judgment' in report + + def test_review_requires_classification_and_rejects_invalid_finding_ids( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 743e7df..f66e48b 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -4,6 +4,7 @@ import csv import json +import re from pathlib import Path from typing import Any @@ -129,8 +130,26 @@ def _metric(caught: int, total: int) -> str: def _md(value: object) -> str: - """Escape source evidence for Markdown without losing line breaks.""" - return str(value).replace("\\", "\\\\").replace("|", "\\|").replace("\n", "
") + """Escape inline text for Markdown, including raw HTML, keeping line breaks.""" + text = str(value).replace("\\", "\\\\").replace("|", "\\|") + text = text.replace("<", "\\<").replace(">", "\\>") + return text.replace("\n", "
") + + +def _evidence_block(evidence: dict[str, str]) -> list[str]: + """Render source evidence verbatim in a collapsible fenced JSON block.""" + body = json.dumps(evidence, ensure_ascii=False, indent=2) + longest = max((len(run) for run in re.findall(r"`+", body)), default=0) + fence = "`" * max(3, longest + 1) + return [ + "
Original defect evidence", + "", + f"{fence}json", + body, + fence, + "", + "
", + ] def write_report( @@ -186,7 +205,6 @@ def write_report( reference = row["reference"] decision = row["decision"] matches = row["matches"] - evidence = _md(json.dumps(reference["raw_evidence"], ensure_ascii=False)) lines.extend( [ f"### {reference['reference_id']} — {_md(reference['problem'])}", @@ -196,7 +214,6 @@ def write_report( f"row {reference['source_row']} ({reference['page_scope']})" ), f"- source_element: {_md(reference['source_element'])}", - f"- Original defect evidence: {evidence}", ( f"- Classification: {decision['classification']} — " f"{_md(decision['classification_reason'])}" @@ -216,5 +233,5 @@ def write_report( f"- {finding_id}: {_md(finding['problem'])} " f"@ {_md(finding['location'])}" ) - lines.append("") + lines.extend(["", *_evidence_block(reference["raw_evidence"]), ""]) path.write_text("\n".join(lines) + "\n", encoding="utf-8") From 12c17ab311ae30e62e208f08784760c5a4449c12 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 17:27:32 -0400 Subject: [PATCH 045/126] docs(evaluation): record plain-language report spec and tickets --- ...ename-to-human-audit-and-llm-vocabulary.md | 23 +++ .../issues/02-plain-language-report.md | 38 +++++ .../issues/03-live-verification-run.md | 15 ++ .scratch/plain-language-report/spec.md | 143 ++++++++++++++++++ CONTEXT.md | 63 +++++--- .../0002-rank-models-by-llm-detection-rate.md | 3 + ...0002-rank-models-by-workbook-row-recall.md | 3 - 7 files changed, 260 insertions(+), 28 deletions(-) create mode 100644 .scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md create mode 100644 .scratch/plain-language-report/issues/02-plain-language-report.md create mode 100644 .scratch/plain-language-report/issues/03-live-verification-run.md create mode 100644 .scratch/plain-language-report/spec.md create mode 100644 docs/adr/0002-rank-models-by-llm-detection-rate.md delete mode 100644 docs/adr/0002-rank-models-by-workbook-row-recall.md diff --git a/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md b/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md new file mode 100644 index 0000000..48508d5 --- /dev/null +++ b/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md @@ -0,0 +1,23 @@ +# 01: Rename to Human audit and LLM vocabulary (clean break) + +**What to build:** An evaluator using the evaluation CLI works entirely in the glossary vocabulary (Human audit, Human finding, LLM finding, Programmatic finding). A new Evaluation run takes the benchmark spreadsheet via `--human-audit` and writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`. Its `review.csv` has `human_finding_id` and `llm_finding_id` columns, and its IDs carry `human-`, `llm-` and `programmatic-` prefixes. The run manifest records `human_audit_filename` and `human_audit_sha256`. CLI help and error messages and the README's evaluation sections use the glossary terms. `report` reads only the new names and still produces the current report layout; the layout is rewritten in ticket 02. There is no compatibility layer and no migration of old run directories. See the Rename decisions in `../spec.md`. + +**Blocked by:** None (can start immediately). + +**Status:** ready-for-agent + +- [ ] The `run` subcommand accepts `--human-audit` in place of `--workbook`, with the default-path constant renamed to match +- [ ] A run writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`, and never writes `references.json`, `raw-audit-responses.json` or `normalized-audit-findings.json` (asserted in tests) +- [ ] `review.csv` uses `human_finding_id` and `llm_finding_id` columns, and `report` validates against those columns +- [ ] Human finding IDs start with `human-`, LLM finding IDs with `llm-`, and Programmatic finding IDs with `programmatic-`; the hash part of each ID is unchanged +- [ ] LLM findings record `source` as `"llm"` +- [ ] The import module/function and records are renamed (`human_audit`, `import_homepage_human_findings`, `HumanFinding`, `HumanAudit` with `human_findings`) +- [ ] The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match +- [ ] The manifest records `human_audit_filename` and `human_audit_sha256`, and `report` checks the Human audit checksum against them +- [ ] CLI help and error messages say "Human audit", "Human finding" and "LLM finding" instead of "Reference workbook", "Reference defect", "Excluded Reference" or "audit finding" +- [ ] The README's evaluation workflow, run-directory file table, review-column guide and troubleshooting text use the new names +- [ ] Classification values, the spreadsheet's sheet and column headers, and the programmatic artifact names are unchanged +- [ ] Evaluation CLI tests pass through the `main` seam with the fake LLM client, and the full test suite passes +- [ ] The commit is marked breaking (`!` and a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported) + +## Comments diff --git a/.scratch/plain-language-report/issues/02-plain-language-report.md b/.scratch/plain-language-report/issues/02-plain-language-report.md new file mode 100644 index 0000000..68fb779 --- /dev/null +++ b/.scratch/plain-language-report/issues/02-plain-language-report.md @@ -0,0 +1,38 @@ +# 02: Plain-language report + +**What to build:** A reader with no knowledge of the evaluator opens `report.md` and learns, in plain language, which Human findings the LLM and the Programmatic checks caught, what the tool said, why each catch counts, what was missed, and what could not be tested. `report` requires `--reviewer`. The report follows the layout in `../spec.md`, in this order: + +1. title; +2. How to read this report; +3. Summary; +4. Human findings table; +5. What each part of the tool caught; +6. Missed; +7. Not testable from the page file. + +It contains no IDs, CSS paths, location JSON, raw evidence JSON or raw usage JSON. Scoring semantics are unchanged. + +**Blocked by:** 01 (Rename to Human audit and LLM vocabulary) + +**Status:** ready-for-agent + +- [ ] `report` fails without `--reviewer`; with it, the Summary shows "Review decisions by: " +- [ ] The title is "LLM accessibility evaluation: ", and the Summary shows the model and run ID +- [ ] "How to read this report" says: + - the benchmark is the Human audit; + - the tool has two parts (the LLM and programmatic checks), each defined; + - the tool saw only the saved HTML; + - a reviewer set expectations and only full coverage counts; + - the scope is the homepage only +- [ ] The LLM, Programmatic and Overall detection rates each appear as a sentence with the count, the denominator explained, and a percentage; zero eligible findings is stated in words +- [ ] The cost line shows dollars rounded to cents, input and output tokens, and the pricing-schedule publication date, and ends "The actual bill may differ slightly." Cached and cache-creation tokens appear only when non-zero +- [ ] A sentence names the LLM checks with malformed output out of the total prompt count, and is omitted when there are none +- [ ] The table has one row per Human finding in source order, with columns for the Issue Title, WCAG, expected part (LLM / Programmatic checks / Not testable), and ✓/✗ for each part; cross-catches are marked, and a footnote explains what the rates count +- [ ] Each caught Human finding lists its matched findings, labelled "LLM ( check)" or "Programmatic check ", with the problem text and "Why this counts:" giving the match reason +- [ ] "Missed" lists every `llm_eligible` or `programmatic` Human finding with no match, even when the classification reason and review notes are both empty; "Why it was expected to be caught" and "Why it was missed" appear only when non-empty +- [ ] "Not testable from the page file" groups `unavailable_evidence` and `ambiguous` findings under one intro sentence, each with its classification reason +- [ ] The report contains no Human finding or finding IDs, `source_element`, CSS paths or JSON blocks, and free text stays escaped for Markdown (inline HTML and table pipes) +- [ ] Tests through the `main` seam cover each item above, including a cross-catch, a missed finding with no explanations, both not-testable classifications, and the zero-eligible wording +- [ ] The README's description of `report` and its output matches the new layout, including the `--reviewer` argument + +## Comments diff --git a/.scratch/plain-language-report/issues/03-live-verification-run.md b/.scratch/plain-language-report/issues/03-live-verification-run.md new file mode 100644 index 0000000..484a79b --- /dev/null +++ b/.scratch/plain-language-report/issues/03-live-verification-run.md @@ -0,0 +1,15 @@ +# 03: Live verification run with an agent reviewer + +**What to build:** The operator sees the new report produced from real model output. One paid Evaluation run is made on the saved Pristine homepage snapshot. A sub-agent acting as the reviewer fills in every `review.csv` row, and the report is generated and read as an outsider would read it. The operator authorized this spend in the planning conversation; follow the spending controls in `.model-evaluation/live-run-plan.md` (sequential requests, no retries, stop on failure). + +**Blocked by:** 02 (Plain-language report) + +**Status:** ready-for-agent + +- [ ] One live run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with `--max-cost-usd 1.10` completes, or its stop reason is reported without a retry +- [ ] A sub-agent fills in every `review.csv` row. It classifies each Human finding against the snapshot before reading the LLM responses, then records Match decisions following the README's rules, with a reason for every classification and match +- [ ] `report` succeeds with `--reviewer "Claude agent (operator-directed)"` +- [ ] The generated report is read end to end, and any wording that still needs knowledge of the evaluator is reported to the operator +- [ ] The run's estimated cost and detection rates are recorded under Comments + +## Comments diff --git a/.scratch/plain-language-report/spec.md b/.scratch/plain-language-report/spec.md new file mode 100644 index 0000000..d041f2d --- /dev/null +++ b/.scratch/plain-language-report/spec.md @@ -0,0 +1,143 @@ +# Plain-language evaluation report and Human audit naming + +Status: ready-for-agent + +## Problem Statement + +A reader of an Evaluation run's final report has to know how the evaluator works before the report makes sense. It uses undefined jargon: "Workbook-row recall", "Combined workbook coverage", "Reference defect", "Audit findings", and raw classification values such as `llm_eligible` and `unavailable_evidence`. Most of its length is spent on material the reader does not need, such as reference and finding IDs, the `source_element` field, CSS paths and location JSON, raw token-usage JSON, and a collapsible raw-evidence JSON block for every row. It never plainly says which defects the human testers reported, which of those the tool caught, which part of the tool caught each one, and which were missed. The report does not define "unavailable evidence" or "ambiguous", so a reader cannot tell why some defects were left out of the scores. + +The same jargon runs through the evaluator's code, artifact files, review CSV columns, CLI flags and README. "Workbook" names the file format of the benchmark rather than what it is: an audit performed by humans. "Audit finding" is ambiguous, because the human testers also produced an audit. + +## Solution + +The report is rewritten for a reader who knows nothing about the evaluator. It opens with a short "How to read this report" section and gives the headline numbers as plain sentences with their denominators explained. A single table then shows every Human finding and whether the LLM, the Programmatic checks, or neither caught it. For each catch the report shows what the tool said and the reviewer's reason the catch counts. Every miss is listed with background on why it was expected to be caught and why it was missed. Findings that could not be tested from the saved page are grouped under one plain heading, "Not testable from the page file", each with its reason. The estimated cost is explained as token counts multiplied by published prices. The report states who made the review decisions, so agent-made decisions are never mistaken for human adjudication. + +The evaluator adopts the glossary in `CONTEXT.md` everywhere, including code, artifact files, CSV columns, CLI flags, error messages and the README. The terms are Human audit, Human finding, LLM, LLM finding, Programmatic checks, Programmatic finding, LLM detection rate, Programmatic detection rate, Overall detection rate, and Not testable from the page file. The rename is a clean break with no migration of old run directories. + +## User Stories + +1. As a report reader with no knowledge of the evaluator, I want a short "How to read this report" section at the top, so that I understand what is being measured before I see any numbers. +2. As a report reader, I want the benchmark called the "Human audit" and each of its defects a "Human finding", so that I know the comparison is against human testers and not a spreadsheet artifact. +3. As a report reader, I want the two parts of the tool named "the LLM" and "programmatic checks", each defined in one sentence, so that I can tell AI judgment from fixed rules. +4. As a report reader, I want to be told that the tool saw only a saved copy of the page's HTML, not a live browser or screen reader, so that I understand why some human findings could not be tested. +5. As a report reader, I want to be told that a catch counts only when the tool's findings cover the whole Human finding, so that I don't read partial matches as successes. +6. As a report reader, I want to be told the evaluation covers the homepage only, so that I don't generalise the numbers to the whole site. +7. As a report reader, I want the LLM detection rate written as a sentence ("Of the 8 Human findings the LLM could be expected to detect from the page's HTML, it caught 3 (37.5%)"), so that I know what the percentage is out of. +8. As a report reader, I want the Programmatic detection rate written the same way, so that I can compare the two parts of the tool on equal terms. +9. As a report reader, I want an Overall detection rate saying how many of all Human findings were caught by either part, so that I see the tool's total reach. +10. As a report reader, I want a rate with zero eligible findings stated in words and not as a division by zero, so that the summary never shows a meaningless percentage. +11. As a report reader, I want to see the model and run ID in the summary, so that I know which configuration produced these results. +12. As a report reader, I want the estimated cost in dollars and cents with the token counts and the price-list date it came from, so that I understand it is an estimate and how it was calculated. +13. As a report reader, I want cached-token counts mentioned only when they are non-zero, so that the cost line stays short in the common case. +14. As a report reader, I want a plain sentence naming any LLM checks that returned unreadable output, so that I know some LLM misses were caused by broken responses and not poor judgment. +15. As a report reader, I want that sentence left out when every LLM check returned readable output, so that I'm not distracted by a "none" line. +16. As a report reader, I want a table with one row per Human finding showing its title, WCAG criterion, which part of the tool it was expected from, and ✓/✗ for each part, so that I see the whole outcome at a glance. +17. As a report reader, I want the table to mark every catch even when the part of the tool that was not expected to catch a finding caught it, so that I see everything the tool actually found. +18. As a report reader, I want a footnote explaining that the detection rates count only catches by the expected part of the tool, so that a cross-catch ✓ that doesn't change a rate doesn't confuse me. +19. As a report reader, I want each caught Human finding to list the matched findings, labelled by source ("LLM (link clarity check)" or "Programmatic check LAND_001") with the tool's problem text, so that I can see what the tool actually said. +20. As a report reader, I want the reviewer's reason for each catch shown as "Why this counts", so that I can judge whether the ✓ is deserved. +21. As a report reader, I want the source labels to make the rule names cited in match reasons understandable, so that a reason like "LAND_001 reports exactly this" makes sense. +22. As a report reader, I want a "Missed" section listing every counted Human finding that nothing caught, so that the tool's gaps are explicit. +23. As a report reader, I want each missed finding to show why it was expected to be caught, so that I believe it was a real miss and not an impossible task. +24. As a report reader, I want each missed finding to show why it was missed when the reviewer recorded a reason, so that I understand the failure. +25. As a report reader, I want a missed finding listed even when neither explanation was recorded, so that no miss is silently hidden. +26. As a report reader, I want findings classified as unavailable evidence or ambiguous grouped under "Not testable from the page file", with one sentence saying they are left out of every detection rate, so that I don't need to learn two internal categories. +27. As a report reader, I want each not-testable finding shown with the reviewer's specific reason, so that I know why it couldn't be tested. +28. As a report reader, I want the report to say who made the review decisions, so that I know whether a human or an agent judged the matches. +29. As a report reader, I want no IDs, CSS paths, location JSON, raw evidence JSON or raw usage JSON in the report, so that the document stays readable. +30. As a report reader, I want Human findings shown in the Human audit's original order, so that I can cross-reference the original spreadsheet. +31. As an evaluator, I want `report` to require a reviewer description, so that I can't publish a report that hides who made the decisions. +32. As an evaluator, I want the CLI flag for the benchmark spreadsheet called `--human-audit`, so that the command reads in the project's vocabulary. +33. As an evaluator, I want the imported Human findings saved as `human-findings.json`, so that the artifact name says what it holds. +34. As an evaluator, I want the review CSV columns called `human_finding_id` and `llm_finding_id`, so that I know which ID goes in which column. +35. As an evaluator, I want LLM artifacts called `raw-llm-responses.json` and `normalized-llm-findings.json`, so that they sit alongside the programmatic `raw-…`/`normalized-…` files under matching names. +36. As an evaluator, I want IDs prefixed `human-`, `llm-` and `programmatic-`, so that I can tell at a glance what kind of item an ID refers to when filling in the review CSV. +37. As an evaluator, I want CLI help and error messages to use the glossary terms ("Human finding", "LLM finding"), so that errors match the vocabulary in the CSV and README. +38. As an evaluator, I want the run manifest to record `human_audit_filename` and `human_audit_sha256`, so that its provenance fields use the same name as the concept. +39. As an evaluator, I want the README's evaluation workflow, file table, review-column guide and report description updated to the new names and layout, so that the instructions match what's on disk. +40. As an evaluator, I want the classification values (`llm_eligible`, `programmatic`, `unavailable_evidence`, `ambiguous`) unchanged in the CSV, so that the reviewer still records the finer distinction even though the report combines two of them. +41. As a maintainer, I want the rename to be a clean break with no compatibility code, so that the evaluator doesn't carry fallbacks for throwaway local output. +42. As a maintainer, I want the change marked as breaking in its commit, so that anyone holding an old run directory knows `report` will no longer read it. +43. As a maintainer, I want tests that fail if old artifact names are written again, so that the rename can't silently regress. +44. As a maintainer, I want scoring rules unchanged, so that the new report's rates equal the old metrics under new names. +45. As the operator, I want the change verified with a real paid run reviewed by an agent acting as the reviewer, so that I can judge the new report against real model output before relying on it. + +## Implementation Decisions + +- **Glossary.** `CONTEXT.md` already holds the resolved terms, and ADR 0002 has been reworded to "LLM detection rate" and renamed accordingly. The ranking decision itself is unchanged. This spec supersedes the "Out of Scope" decision in the `rename-audit-artifacts` spec that kept the name "Audit finding". +- **Report module.** The report writer in the review module is rewritten. Its inputs are the same (the manifest, reviewed rows, and LLM and Programmatic findings) plus the reviewer description and the pricing-schedule date. It emits these sections in order: + 1. title "LLM accessibility evaluation: "; + 2. How to read this report; + 3. Summary; + 4. Human findings table; + 5. What each part of the tool caught; + 6. Missed; + 7. Not testable from the page file. +- **Scoring semantics unchanged.** + - LLM detection rate: `llm_eligible` rows with at least one matched LLM finding, out of all `llm_eligible` rows. + - Programmatic detection rate: `programmatic` rows with at least one matched Programmatic finding, out of all `programmatic` rows. + - Overall detection rate: rows with any match, out of all imported Human findings, so not-testable findings stay in that denominator. + - Review validation rules are unchanged, including that not-testable rows cannot claim matches. +- **"Expected to be caught by" mapping.** `llm_eligible` shows as "LLM", `programmatic` as "Programmatic checks", and `unavailable_evidence`/`ambiguous` as "Not testable". +- **Missed definition.** A missed finding is an `llm_eligible` or `programmatic` Human finding with no matched finding of either kind. It is always listed. "Why it was expected to be caught" (the classification reason) and "Why it was missed" (the review notes) appear only when non-empty. +- **Finding source labels.** LLM findings are labelled "LLM ( check)". Programmatic findings are labelled "Programmatic check ". Both come from the finding's recorded prompt field. +- **Cost line.** + - The dollar amount is the run's estimated cost rounded to cents; the exact figure stays in the manifest. + - Input and output token counts are always shown. Cached and cache-creation counts are shown only when non-zero. + - The price date is the recorded pricing schedule's publication date, and the line ends "The actual bill may differ slightly." +- **Malformed-output sentence.** It names the LLM checks listed as format failures in the manifest, in words, out of the total prompt count, and is omitted when there are none. +- **Markdown safety.** Free text from the Human audit, review CSV and findings is still escaped for inline Markdown, including raw HTML and table pipes. +- **Reviewer attribution.** The `report` subcommand gains a required `--reviewer` text argument, printed in the summary as "Review decisions by: ". +- **Rename (clean break, no migration or compatibility layer):** + - CLI flag `--workbook` → `--human-audit`, and the default path constant renamed to match. + - Artifact `references.json` → `human-findings.json`. + - Human audit import module and function renamed to `human_audit` / `import_homepage_human_findings`. + - Records `ReferenceDefect` → `HumanFinding` and `ReferenceSet` → `HumanAudit`; its collection field becomes `human_findings`. + - Field and CSV column `reference_id` → `human_finding_id`; CSV column `audit_finding_id` → `llm_finding_id`. + - Manifest and HumanAudit fields `workbook_filename`/`workbook_sha256` → `human_audit_filename`/`human_audit_sha256`. + - Artifacts `raw-audit-responses.json` → `raw-llm-responses.json` and `normalized-audit-findings.json` → `normalized-llm-findings.json`. + - Finding `source` value `"audit"` → `"llm"`. + - The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match. + - ID prefixes `ref-` → `human-` and `finding-` → `llm-`; `programmatic-` is unchanged. The hash portion of each ID is unchanged. + - User-facing CLI help text and error messages use the glossary terms. + - README evaluation sections are updated to the new names and report layout. +- **Unchanged:** the classification values, the Human audit's sheet name and column headers (source data), the programmatic artifact names, the prompt/payload/snapshot/pricing artifacts, and the shared pipeline used by the web server. +- **Commit** as `refactor(evaluation)!:`/`feat(evaluation)!:` with a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported. + +## Testing Decisions + +- A good test drives the evaluator only through its external seam: the evaluation CLI's `main` entry point, with the LLM request client replaced by the existing fake. Tests assert on the files in the run directory and the text of `report.md`, never on internal helpers. This is the single seam, and it already exists. +- Prior art: the existing evaluation CLI tests already build a small Human audit spreadsheet and HTML input, run live with a fake client, edit `review.csv`, and assert on `report.md`. They also assert that old artifact names are absent after a run. +- Update the existing report assertions to the new rate sentences and names, e.g. LLM detection rate 1/1, Programmatic detection rate 1/1, Overall detection rate 2/3, and the zero-eligible wording. +- New assertions through the same seam: + - The table marks a cross-catch: a `programmatic`-classified finding matched by an LLM finding shows ✓ under LLM without changing the LLM detection rate. + - A missed finding with empty classification reason and review notes is still listed under "Missed". + - Both `unavailable_evidence` and `ambiguous` findings appear under "Not testable from the page file" with their reasons. + - Matched findings show their source label, problem text and "Why this counts". + - The report contains no Human finding IDs, finding IDs, CSS paths or JSON evidence blocks. + - `report` fails without `--reviewer`, and prints it when given. + - The cost line shows cents and the pricing date, and the malformed-output sentence appears only when there are format failures. + - Inline HTML in free text is still escaped. + - After a run, the new artifact names exist and the old ones (`references.json`, `raw-audit-responses.json`, `normalized-audit-findings.json`) do not; review CSV columns use the new names; IDs use the new prefixes. +- No new unit tests below the CLI seam. +- **Live verification (operator-requested, paid):** + 1. After the tests pass, execute one live Evaluation run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with a between-request cost guardrail of $1.10. Previous identical runs cost about $0.11. + 2. A sub-agent acting as the reviewer fills in every `review.csv` row. It classifies against the snapshot independently of the LLM responses, then records Match decisions using the README's rules. + 3. Generate the report with `--reviewer "Claude agent (operator-directed)"` and read it as an outsider would. + +## Out of Scope + +- Changing classification values, scoring rules, review validation rules, or the model ranking decision. +- Migrating, reading or deleting existing local run directories. +- Renaming the Human audit spreadsheet's sheet or columns. +- Changes to the shared pipeline, the web server, or `run_pipeline.py`. +- Precision or other metrics beyond the three detection rates. + +## Further Notes + +- Considered and rejected: showing "unavailable evidence" and "ambiguous" as two report categories. Readers need to know why each finding wasn't counted, and the per-row reasons say that better than category names. The CSV keeps the distinction. +- Considered and rejected: showing ✓/✗ without the tool's text. A ✓ is only believable if the reader can see what the tool said and why it counts. +- Considered and rejected: a migration script or a compatibility layer for old runs. The operator no longer needs them. +- The estimated cost is not an invoice: token counts are exact as reported by the provider, but dollar amounts depend on the recorded public price schedule. + +## Comments diff --git a/CONTEXT.md b/CONTEXT.md index ca2be15..551f88b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -4,52 +4,65 @@ This context names the evidence and measures for the homepage accessibility expe ## Language -**Reference workbook**: -The authorized human accessibility audit workbook that supplies source evidence. -_Avoid_: Ground-truth spreadsheet, generated review workbook. +**Human audit**: +The authorized accessibility audit performed by human testers and delivered as an Excel spreadsheet. It supplies the source evidence the tool is measured against. +_Avoid_: Reference workbook, workbook, ground-truth spreadsheet, generated review workbook. -**Reference defect**: -One source workbook defect retained with its original evidence and sheet and row provenance. Its `source_element` is the original element name, not a precise inferred location. -_Avoid_: Expected finding, inferred location. +**Human finding**: +One accessibility defect from the Human audit, kept with its original evidence and with the sheet and row it came from. Its `source_element` is the element name the testers wrote, not a precise inferred location. +_Avoid_: Reference defect, workbook row, expected finding, inferred location. **Benchmark snapshot**: -The identified HTML evidence for the page being audited, paired with its source URL and checksum. +The saved HTML of the audited page, identified by its source URL and checksum. _Avoid_: Live page. **Evaluation run**: -One fixed-model audit and its evidence, human decisions, completion status, and report. +One audit with a fixed model, together with its evidence, the human decisions, its completion status, and its report. _Avoid_: Execution plan, report bundle. -**Audit finding**: -An accessibility problem reported by the AI audit model and retained with its source response. -_Avoid_: Reference defect, programmatic finding. +**LLM**: +The part of the tool that asks a large language model to judge the page for accessibility problems. +_Avoid_: Audit model, AI audit. + +**Programmatic checks**: +The part of the tool that applies fixed rules to the page's HTML. It uses no AI and gives the same answer every time. +_Avoid_: Deterministic checker. + +**LLM finding**: +An accessibility problem reported by the LLM, kept with the response it came from. +_Avoid_: Audit finding, Human finding, Programmatic finding. **Programmatic finding**: -An accessibility problem reported by a deterministic checker rather than the AI audit model. -_Avoid_: Audit finding. +An accessibility problem reported by the Programmatic checks. +_Avoid_: Audit finding, LLM finding. **Eligibility classification**: -The human assignment of a Reference defect to `llm_eligible`, `programmatic`, `unavailable_evidence`, or `ambiguous`, with a reason. +A reviewer's decision about which part of the tool should be expected to catch a Human finding, recorded with a reason. The values are `llm_eligible`, `programmatic`, `unavailable_evidence` (the Benchmark snapshot cannot show the problem), and `ambiguous` (the Human finding is too unclear to judge). _Avoid_: Automatic model score, exclusion without rationale. +**Not testable from the page file**: +The reader-facing group for Human findings classified `unavailable_evidence` or `ambiguous`. They are left out of every detection rate, and each one is reported with its reason. +_Avoid_: Excluded, unavailable, ambiguous (in reports). + **Match decision**: -A human judgment that selected findings jointly provide full-row coverage of one Reference defect. Partial evidence alone does not award credit. +A reviewer's judgment that the selected findings together fully cover one Human finding. Partial evidence alone earns no credit. _Avoid_: Candidate match, partial-credit score. -**Workbook-row recall**: -The proportion of LLM-eligible Reference defects fully covered by Audit findings. -_Avoid_: Precision, F1, aggregate quality score. +**LLM detection rate**: +The proportion of `llm_eligible` Human findings that LLM findings fully cover. +_Avoid_: Workbook-row recall, precision, F1, aggregate quality score. -**Programmatic coverage**: -The proportion of programmatic Reference defects fully covered by Programmatic findings. Classification alone does not award coverage. +**Programmatic detection rate**: +The proportion of `programmatic` Human findings that Programmatic findings fully cover. A classification on its own earns no coverage. +_Avoid_: Programmatic coverage. -**Combined workbook coverage**: -The proportion of imported Reference defects fully covered by either finding kind, counting each row once. -_Avoid_: Workbook-row recall, model score. +**Overall detection rate**: +The proportion of all imported Human findings that either kind of finding fully covers, counting each Human finding once. +_Avoid_: Combined workbook coverage, LLM detection rate, model score. **Estimated run cost**: -The cost calculated from reported usage and a recorded pricing schedule, rather than an invoice amount. +The token counts the provider reported, multiplied by a recorded public price schedule. It is not an invoice amount. **Live-run approval**: -An operator's explicit authorization of a displayed fixed-model execution under a positive between-request cost guardrail. +An operator's explicit authorization to run a displayed fixed-model execution, given with a positive cost guardrail that is checked between requests. _Avoid_: API-key presence. diff --git a/docs/adr/0002-rank-models-by-llm-detection-rate.md b/docs/adr/0002-rank-models-by-llm-detection-rate.md new file mode 100644 index 0000000..a6628be --- /dev/null +++ b/docs/adr/0002-rank-models-by-llm-detection-rate.md @@ -0,0 +1,3 @@ +# Rank models by LLM detection rate + +Models are ranked first by LLM detection rate over human-approved `llm_eligible` Human findings, with estimated run cost as the only tie-breaker; latency, token usage, unmatched findings, programmatic detection rate, and overall detection rate are reported but do not change rank. This keeps the Human audit as the principal measure while preventing model-independent programmatic findings or an arbitrary composite formula from obscuring the model comparison. diff --git a/docs/adr/0002-rank-models-by-workbook-row-recall.md b/docs/adr/0002-rank-models-by-workbook-row-recall.md deleted file mode 100644 index ed4fd57..0000000 --- a/docs/adr/0002-rank-models-by-workbook-row-recall.md +++ /dev/null @@ -1,3 +0,0 @@ -# Rank models by workbook-row recall - -Models are ranked first by workbook-row recall over human-approved `llm_eligible` rows, with estimated audit-run cost as the only tie-breaker; latency, token usage, unmatched findings, programmatic coverage, and combined workbook coverage are reported but do not change rank. This keeps the mandated defect workbook as the principal measure while preventing model-independent programmatic findings or an arbitrary composite formula from obscuring the model comparison. From 401822c9a4a60ac4e196f0bef53e07432990aed5 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 17:32:53 -0400 Subject: [PATCH 046/126] refactor(evaluation)!: adopt Human audit and LLM vocabulary Rename evaluator inputs, evidence artifacts, finding IDs and review fields while preserving report layout and scoring. BREAKING CHANGE: Existing Evaluation run directories are unsupported; start a new run using --human-audit. --- README.md | 59 ++++---- tests/test_evaluation_cli.py | 134 +++++++++++------- vision_aid/evaluation/cli.py | 38 ++--- .../{references.py => human_audit.py} | 26 ++-- .../evaluation/{audit.py => llm_audit.py} | 13 +- vision_aid/evaluation/normalization.py | 6 +- vision_aid/evaluation/review.py | 94 ++++++------ vision_aid/evaluation/schemas.py | 14 +- 8 files changed, 208 insertions(+), 176 deletions(-) rename vision_aid/evaluation/{references.py => human_audit.py} (68%) rename vision_aid/evaluation/{audit.py => llm_audit.py} (92%) diff --git a/README.md b/README.md index e5a6dc4..af8e8ce 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ GEMINI_API_KEY=AIza... ## Private Model Evaluation -### Get a final report with the supplied Reference workbook +### Get a final report with the supplied Human audit From the repository root, use the supplied `Pristine Accessibility Defect Report.xlsx` (16 Home and Global defects). These steps work after deleting `.model-evaluation/`. @@ -268,6 +268,7 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report ```bash MAX_COST_USD=1.00 # Example; replace with the amount you approve. uv run visionaid-evaluate run \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ --html .model-evaluation/pristine-home.html \ --source-url https://pristineai.com/ \ --max-cost-usd "$MAX_COST_USD" \ @@ -277,7 +278,7 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report The CLI shows Haiku's 16,000 thinking tokens, 24,192 output token cap, prompt count, pricing, snapshot checksum, and limit. Type `yes` to approve paid requests; `--approve-live` skips the prompt. It copies the snapshot, - imports Reference defects, runs checks, and saves responses and usage. + imports Human findings, runs checks, and saves responses and usage. 4. **Check completion and review the findings.** Use the printed run directory: @@ -287,13 +288,13 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report Confirm `"complete": true` in `$RUN_DIR/run.json`; otherwise, address the failure or cost limit and start a new live run. Failed requests are not - retried. Use `$RUN_DIR/normalized-audit-findings.json` and + retried. Use `$RUN_DIR/normalized-llm-findings.json` and `$RUN_DIR/normalized-programmatic-findings.json` for finding IDs and evidence. - `$RUN_DIR/raw-audit-responses.json` has model responses; + `$RUN_DIR/raw-llm-responses.json` has model responses; `$RUN_DIR/raw-programmatic-findings.json` has the original checker output. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in - Excel or a CSV-aware text editor. Review every Reference defect and fill in + Excel or a CSV-aware text editor. Review every Human finding and fill in the decision columns using the guide below. Save the file as CSV. 6. **Generate the final report.** Run `report` after human review: @@ -302,10 +303,10 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report uv run visionaid-evaluate report --run-dir "$RUN_DIR" ``` - Open `$RUN_DIR/report.md` for Workbook-row recall, programmatic coverage, - Combined workbook coverage, Estimated run cost, limitations, and source-row - explanations. Reporting rejects incomplete runs, missing decisions, and - invalid or reused finding IDs. + Open `$RUN_DIR/report.md` for LLM detection rate, Programmatic detection rate, + Overall detection rate, Estimated run cost, limitations, and source-row + explanations. Existing Evaluation run directories are unsupported. Reporting + rejects incomplete runs, missing decisions, and invalid or reused finding IDs. ### Evaluation run files @@ -314,23 +315,23 @@ prepares the prompts but makes no model requests. | File | Role | | --- | --- | -| `references.json` | The imported Home and Global Reference defects, their original workbook evidence, and stable IDs. | +| `human-findings.json` | The imported Home and Global Human findings, their original Human audit evidence, and stable IDs. | | `payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | | `payloads/cl02_payload.json` | Extracted forms, fields, and labels. | | `payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | | `prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | -| `raw-programmatic-findings.json` | Original output from the deterministic accessibility checks. | +| `raw-programmatic-findings.json` | Original output from the Programmatic checks. | | `normalized-programmatic-findings.json` | Those checker results converted to findings with `finding_id` values for `review.csv`. | -| `raw-audit-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | -| `normalized-audit-findings.json` | Parsed AI Audit findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-audit-responses.json` without producing a finding here. | +| `raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | +| `normalized-llm-findings.json` | Parsed LLM findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-llm-responses.json` without producing a finding here. | | `run.json` | The run's configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | | `pricing.json` | A copy of the versioned model rates used to estimate this run's cost from reported usage. | The payload files show extracted page data before filtering. The prompt files show what the model actually received after filtering and slicing. To review a -Reference defect, compare `references.json` with the two normalized findings +Human finding, compare `human-findings.json` with the two normalized findings files and inspect `snapshot.html` for the full Benchmark evidence. Check -`raw-audit-responses.json` when an AI finding is missing or malformed, and +`raw-llm-responses.json` when an LLM finding is missing or malformed, and check the prompt file to see whether the model received the relevant evidence. Record decisions in `review.csv`; the `report` command reads that CSV and the run evidence to write `report.md`. The pipeline briefly writes `manifest.json`, @@ -345,24 +346,24 @@ create those files or perform those extra checks. ### Fill in `review.csv` -The CSV has one row for each of the 16 Home or Global Reference defects. Keep -the header, all rows, and their `reference_id` values. The columns from -`reference_id` through `comment` identify the workbook row and preserve its +The CSV has one row for each of the 16 Home or Global Human findings. Keep +the header, all rows, and their `human_finding_id` values. The columns from +`human_finding_id` through `comment` identify the Human finding and preserve its original evidence. Use them to understand the defect, but do not edit them. -In particular, `source_sheet` and `source_row` point back to the workbook, -`page_scope` says Home or Global, and `source_element` copies the workbook's +In particular, `source_sheet` and `source_row` point back to the Human audit, +`page_scope` says Home or Global, and `source_element` copies the Human audit's `element name`. It is not an inferred location on the page. For **every row**, fill in `classification` and `classification_reason`: | `classification` value | Use it when | | --- | --- | -| `llm_eligible` | The Benchmark snapshot contains enough evidence to judge the defect, and the issue calls for the AI audit's judgment. | -| `programmatic` | A deterministic check can establish the defect from the available evidence. Classification alone does not mean the checker found it. | +| `llm_eligible` | The Benchmark snapshot contains enough evidence to judge the defect, and the issue calls for the LLM's judgment. | +| `programmatic` | A Programmatic check can establish the defect from the available evidence. Classification alone does not mean the checker found it. | | `unavailable_evidence` | The snapshot lacks evidence needed to verify the defect, such as behavior that requires interaction or a browser or assistive technology observation. | -| `ambiguous` | The Reference defect is too unclear to decide fairly whether a finding covers it. | +| `ambiguous` | The Human finding is too unclear to decide fairly whether a finding covers it. | -Base the classification on the Reference defect and available evidence, not on +Base the classification on the Human finding and available evidence, not on whether this run happened to produce a matching finding. In `classification_reason`, write a short explanation of that choice, including what evidence is missing or unclear when applicable. These two cells cannot be @@ -372,19 +373,19 @@ Then complete the remaining decision columns for that row: | Column | What to enter | | --- | --- | -| `audit_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-audit-findings.json` only when the selected AI Audit findings **jointly cover the whole Reference defect**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | -| `programmatic_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-programmatic-findings.json` only when the selected Programmatic findings **jointly cover the whole Reference defect**. Use semicolons for multiple IDs. Otherwise leave blank. | +| `llm_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-llm-findings.json` only when the selected LLM findings **jointly cover the whole Human finding**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | +| `programmatic_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-programmatic-findings.json` only when the selected Programmatic findings **jointly cover the whole Human finding**. Use semicolons for multiple IDs. Otherwise leave blank. | | `match_reason` | If either finding-ID cell has an ID, briefly explain how those findings cover the full defect, including the relevant element or behavior. If both ID cells are blank, this can be blank. | | `review_notes` | Optional context, such as a more precise location or a finding that covers only part of the defect. Notes do not award coverage. | You may use findings from both files for one row. Each finding ID must exist in -the named file and can be assigned to only one Reference defect. Leave both ID +the named file and can be assigned to only one Human finding. Leave both ID cells blank when no findings fully cover the row; that records a miss, even if you describe partial evidence in `review_notes`. For `unavailable_evidence` and `ambiguous` rows, leave both ID cells blank. The report command checks these rules and requires a `match_reason` whenever you enter an ID. -The run directory and workbook are excluded from version control. +The run directory and Human audit are excluded from version control. This one-homepage result does not establish broad model equivalence. ### Start a run without `--live` @@ -398,7 +399,7 @@ uv run visionaid-evaluate run --html .model-evaluation/pristine-home.html \ This needs no API key or approval. It creates an incomplete run with `raw-programmatic-findings.json`, `normalized-programmatic-findings.json`, -audit prompts, and `review.csv`. Use the normalized file's `finding_id` values +LLM prompts, and `review.csv`. Use the normalized file's `finding_id` values for programmatic matches. It makes no Haiku requests and cannot produce a final report. `--max-cost-usd` must be positive but is not spent without `--live`. Start a new live run at step 3 to finish the evaluation. diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 78edd07..a92a6df 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -8,12 +8,12 @@ import openpyxl import pytest -from vision_aid.evaluation import audit +from vision_aid.evaluation import llm_audit from vision_aid.evaluation.cli import main def inputs(tmp_path: Path) -> tuple[Path, Path]: - """Build a small workbook with the exact supported source columns.""" + """Build a small Human audit with the exact supported source columns.""" book = openpyxl.Workbook() sheet = book.active sheet.title = "Defect report" @@ -82,22 +82,22 @@ def inputs(tmp_path: Path) -> tuple[Path, Path]: ] ) sheet.append([4, "Other", "Chrome", "Careers", "Outside scope"]) - workbook = tmp_path / "reference.xlsx" - book.save(workbook) + human_audit = tmp_path / "human-audit.xlsx" + book.save(human_audit) html = tmp_path / "home.html" html.write_text( 'Home' '

Home

Click here' ) - return workbook, html + return human_audit, html -def args(workbook: Path, html: Path, *extra: str) -> list[str]: +def args(human_audit: Path, html: Path, *extra: str) -> list[str]: """Assemble the public run command for the synthetic inputs.""" return [ "run", - "--workbook", - str(workbook), + "--human-audit", + str(human_audit), "--html", str(html), "--source-url", @@ -127,17 +127,17 @@ def write_review(path: Path, rows: list[dict[str, str]]) -> None: writer.writerows(rows) -def test_preview_preserves_reference_rows_and_never_calls_provider( +def test_preview_preserves_human_findings_and_never_calls_provider( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: """Credentials alone never buy requests, and all source rows survive.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "available") monkeypatch.setattr( - audit, "AuditRequestClient", lambda **_: pytest.fail("paid call") + llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call") ) - assert main(args(workbook, html)) == 0 + assert main(args(human_audit, html)) == 0 output = capsys.readouterr().out run = directory(tmp_path) assert "claude-haiku-4-5-20251001" in output @@ -150,11 +150,24 @@ def test_preview_preserves_reference_rows_and_never_calls_provider( assert not (run / "programmatic-findings.json").exists() manifest = json.loads((run / "run.json").read_text()) assert manifest["snapshot_sha256"] == hashlib.sha256(html.read_bytes()).hexdigest() + assert manifest["human_audit_filename"] == human_audit.name + assert manifest["human_audit_sha256"] == hashlib.sha256(human_audit.read_bytes()).hexdigest() + imported = json.loads((run / "human-findings.json").read_text()) + assert imported["human_audit_sha256"] == manifest["human_audit_sha256"] + assert all(item["human_finding_id"].startswith("human-") for item in imported["human_findings"]) rows = read_review(run / "review.csv") assert [row["source_row"] for row in rows] == ["2", "3", "4"] assert rows[0]["source_element"] == "First link" assert rows[0]["actual result"] == "Vague link\nfor everyone" assert all(not row["classification"] for row in rows) + assert {"human_finding_id", "llm_finding_id"} <= rows[0].keys() + assert not {"reference_id", "audit_finding_id"} & rows[0].keys() + with pytest.raises(SystemExit) as help_exit: + main(["run", "--help"]) + assert help_exit.value.code == 0 + assert "Human audit" in capsys.readouterr().out + with pytest.raises(SystemExit): + main([*args(human_audit, html), "--workbook", str(human_audit)]) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) @@ -163,12 +176,12 @@ def test_subdirectory_invocation_uses_ignored_worktree_run_root( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A run from a repository subdirectory stays under the root ignore rule.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) (tmp_path / ".git").mkdir() child = tmp_path / "entry_points" child.mkdir() monkeypatch.chdir(child) - assert main(args(workbook, html)) == 0 + assert main(args(human_audit, html)) == 0 assert (tmp_path / ".model-evaluation" / "runs").is_dir() assert not (child / ".model-evaluation").exists() @@ -177,7 +190,7 @@ def test_live_csv_review_and_report_full_row_metrics( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """One fake live run yields hand-calculated metrics and source evidence.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -209,27 +222,30 @@ def client(**options: object) -> FakeClient: client_options.append(options) return fake - monkeypatch.setattr(audit, "AuditRequestClient", client) - assert main(args(workbook, html, "--live", "--approve-live")) == 0 + monkeypatch.setattr(llm_audit, "LLMRequestClient", client) + assert main(args(human_audit, html, "--live", "--approve-live")) == 0 assert client_options[0]["max_retries"] == 0 run = directory(tmp_path) assert fake.calls == 3 assert ( json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" ) - assert (run / "raw-audit-responses.json").exists() - assert not (run / "raw-responses.json").exists() - assert not (run / "audit-findings.json").exists() - findings = json.loads((run / "normalized-audit-findings.json").read_text()) + assert (run / "raw-llm-responses.json").exists() + assert all(not (run / name).exists() for name in ( + "references.json", "raw-audit-responses.json", "normalized-audit-findings.json" + )) + findings = json.loads((run / "normalized-llm-findings.json").read_text()) programmatic_file = run / "normalized-programmatic-findings.json" programmatic = json.loads(programmatic_file.read_text()) assert findings and programmatic + assert all(item["source"] == "llm" and item["finding_id"].startswith("llm-") for item in findings) + assert all(item["finding_id"].startswith("programmatic-") for item in programmatic) review = run / "review.csv" rows = read_review(review) rows[0].update( classification="llm_eligible", classification_reason="Model can see link", - audit_finding_id=findings[0]["finding_id"], + llm_finding_id=findings[0]["finding_id"], match_reason="Full issue", ) rows[1].update( @@ -244,21 +260,31 @@ def client(**options: object) -> FakeClient: write_review(review, rows) assert main(["report", "--run-dir", str(run)]) == 0 report = (run / "report.md").read_text() - assert "Workbook-row recall: 1/1" in report - assert "Programmatic coverage: 1/1" in report - assert "Combined workbook coverage: 2/3" in report + assert "LLM detection rate: 1/1" in report + assert "Programmatic detection rate: 1/1" in report + assert "Overall detection rate: 2/3" in report assert "Vague link" in report and "First link" in report + for row in rows: + row["audit_finding_id"] = row.pop("llm_finding_id") + write_review(review, rows) + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run)]) + imported = json.loads((run / "human-findings.json").read_text()) + imported["human_audit_sha256"] = "changed" + (run / "human-findings.json").write_text(json.dumps(imported)) + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run)]) def test_report_shows_evidence_verbatim_and_escapes_inline_html( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Fenced evidence stays verbatim and inline fields escape raw HTML.""" - workbook, html = inputs(tmp_path) - book = openpyxl.load_workbook(workbook) + human_audit, html = inputs(tmp_path) + book = openpyxl.load_workbook(human_audit) actual = 'Use
tag.\nSay "Main" ```here```' book.active["G2"] = actual - book.save(workbook) + book.save(human_audit) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -271,8 +297,8 @@ def call(self, prompt: str) -> dict: "usage": {"input_tokens": 1, "output_tokens": 1}, } - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: FakeClient()) - assert main(args(workbook, html, "--live", "--approve-live")) == 0 + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) + assert main(args(human_audit, html, "--live", "--approve-live")) == 0 run = directory(tmp_path) review = run / "review.csv" rows = read_review(review) @@ -313,7 +339,7 @@ def test_review_requires_classification_and_rejects_invalid_finding_ids( capsys: pytest.CaptureFixture[str], ) -> None: """Review validation blocks missing decisions, wrong kinds, and reuse.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -326,13 +352,13 @@ def call(self, prompt: str) -> dict: "usage": {"input_tokens": 1, "output_tokens": 1}, } - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: FakeClient()) - assert main(args(workbook, html, "--live", "--approve-live")) == 0 + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) + assert main(args(human_audit, html, "--live", "--approve-live")) == 0 run = directory(tmp_path) review = run / "review.csv" - audit_ids = [ + llm_ids = [ item["finding_id"] - for item in json.loads((run / "normalized-audit-findings.json").read_text()) + for item in json.loads((run / "normalized-llm-findings.json").read_text()) ] programmatic_id = json.loads( (run / "normalized-programmatic-findings.json").read_text() @@ -343,7 +369,7 @@ def call(self, prompt: str) -> dict: rows[0].update( classification="llm_eligible", classification_reason="Visible", - audit_finding_id="; ".join(audit_ids[:2]), + llm_finding_id="; ".join(llm_ids[:2]), programmatic_finding_id=programmatic_id, match_reason="Together cover the row", review_notes="Partial alone", @@ -352,47 +378,47 @@ def call(self, prompt: str) -> dict: rows[2].update(classification="ambiguous", classification_reason="Unclear claim") write_review(review, rows) assert main(["report", "--run-dir", str(run)]) == 0 - assert "Combined workbook coverage: 1/3" in (run / "report.md").read_text() - rows[1]["audit_finding_id"] = audit_ids[0] + assert "Overall detection rate: 1/3" in (run / "report.md").read_text() + rows[1]["llm_finding_id"] = llm_ids[0] rows[1]["match_reason"] = "Conflicts" write_review(review, rows) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) - assert "Conflicting audit finding ID" in capsys.readouterr().err - rows[1]["audit_finding_id"] = "finding-unknown" + assert "Conflicting LLM finding ID" in capsys.readouterr().err + rows[1]["llm_finding_id"] = "llm-unknown" write_review(review, rows) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) - assert "Unknown audit finding ID" in capsys.readouterr().err - rows[1]["audit_finding_id"] = programmatic_id + assert "Unknown LLM finding ID" in capsys.readouterr().err + rows[1]["llm_finding_id"] = programmatic_id write_review(review, rows) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) - assert "Unknown audit finding ID" in capsys.readouterr().err - rows[0]["audit_finding_id"] = "" + assert "Unknown LLM finding ID" in capsys.readouterr().err + rows[0]["llm_finding_id"] = "" rows[0]["programmatic_finding_id"] = "" rows[0]["match_reason"] = "" rows[0]["classification"] = "ambiguous" - rows[1]["audit_finding_id"] = "" + rows[1]["llm_finding_id"] = "" rows[1]["match_reason"] = "" rows[2]["classification"] = "unavailable_evidence" write_review(review, rows) assert main(["report", "--run-dir", str(run)]) == 0 assert ( - "Workbook-row recall: N/A (0 eligible rows)" in (run / "report.md").read_text() + "LLM detection rate: N/A (0 eligible rows)" in (run / "report.md").read_text() ) - assert "Combined workbook coverage: 0/3" in (run / "report.md").read_text() + assert "Overall detection rate: 0/3" in (run / "report.md").read_text() def test_live_execution_needs_explicit_approval( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A noninteractive live flag without approval makes no paid request.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "available") monkeypatch.setattr( - audit, "AuditRequestClient", lambda **_: pytest.fail("paid call") + llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call") ) class DeclinedInput: @@ -401,7 +427,7 @@ def isatty(self) -> bool: return False monkeypatch.setattr("sys.stdin", DeclinedInput()) - assert main(args(workbook, html, "--live")) == 0 + assert main(args(human_audit, html, "--live")) == 0 assert ( json.loads((directory(tmp_path) / "run.json").read_text())["complete"] is False ) @@ -412,7 +438,7 @@ def test_failed_or_over_budget_runs_keep_usage_and_cannot_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool ) -> None: """Failure and budget exhaustion stop after one request and remain inspectable.""" - workbook, html = inputs(tmp_path) + human_audit, html = inputs(tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") @@ -430,14 +456,14 @@ def call(self, prompt: str) -> dict: } fake = FakeClient() - monkeypatch.setattr(audit, "AuditRequestClient", lambda **_: fake) - assert main(args(workbook, html, "--live", "--approve-live")) == 1 + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: fake) + assert main(args(human_audit, html, "--live", "--approve-live")) == 1 run = directory(tmp_path) manifest = json.loads((run / "run.json").read_text()) assert fake.calls == 1 assert manifest["complete"] is False assert manifest["usage"]["input_tokens"] == 100000 assert float(manifest["estimated_cost_usd"]) > 0 - assert len(json.loads((run / "raw-audit-responses.json").read_text())) == 1 + assert len(json.loads((run / "raw-llm-responses.json").read_text())) == 1 with pytest.raises(SystemExit): main(["report", "--run-dir", str(run)]) diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 8327911..68981b3 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -17,7 +17,8 @@ from dotenv import load_dotenv from entry_points.run_pipeline import run_pipeline -from vision_aid.evaluation.audit import ( +from vision_aid.evaluation.human_audit import import_homepage_human_findings +from vision_aid.evaluation.llm_audit import ( CONFIGURATION, MODEL, REQUEST_CONFIG, @@ -27,11 +28,10 @@ ) from vision_aid.evaluation.normalization import normalize_programmatic_findings from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule -from vision_aid.evaluation.references import import_homepage_references from vision_aid.evaluation.review import create_review, reviewed_rows, write_report PROJECT_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_WORKBOOK = PROJECT_ROOT / "Pristine Accessibility Defect Report.xlsx" +DEFAULT_HUMAN_AUDIT = PROJECT_ROOT / "Pristine Accessibility Defect Report.xlsx" def build_parser() -> argparse.ArgumentParser: @@ -41,7 +41,9 @@ def build_parser() -> argparse.ArgumentParser: run = commands.add_parser( "run", help="preview or execute the fixed Haiku experiment" ) - run.add_argument("--workbook", type=Path, default=DEFAULT_WORKBOOK) + run.add_argument( + "--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT, help="Human audit" + ) run.add_argument("--html", type=Path, required=True, help="local Benchmark HTML") run.add_argument("--source-url", required=True, help="URL represented by the HTML") run.add_argument( @@ -86,12 +88,12 @@ def _run(args: argparse.Namespace) -> int: raise ValueError("--approve-live requires --live") if not args.html.is_file(): raise ValueError(f"Benchmark HTML not found: {args.html}") - references = import_homepage_references(args.workbook, args.source_url) + human_audit = import_homepage_human_findings(args.human_audit, args.source_url) run = _new_run_directory() shutil.copyfile(args.html, run / "snapshot.html") shutil.copyfile(DEFAULT_SCHEDULE, run / "pricing.json") snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() - write_json(run / "references.json", references.to_dict()) + write_json(run / "human-findings.json", human_audit.to_dict()) production = run_pipeline( html_path=str(run / "snapshot.html"), output_dir=run, @@ -114,13 +116,13 @@ def _run(args: argparse.Namespace) -> int: run / "normalized-programmatic-findings.json", [asdict(item) for item in findings], ) - create_review(run / "review.csv", list(references.to_dict()["references"])) + create_review(run / "review.csv", list(human_audit.to_dict()["human_findings"])) schedule = PriceSchedule.load(run / "pricing.json") manifest = { "run_id": run.name, "source_url": args.source_url, - "workbook_filename": references.workbook_filename, - "workbook_sha256": references.workbook_sha256, + "human_audit_filename": human_audit.human_audit_filename, + "human_audit_sha256": human_audit.human_audit_sha256, "snapshot_sha256": snapshot_hash, "configuration": CONFIGURATION, "pricing_version": schedule.version, @@ -159,7 +161,7 @@ def _run(args: argparse.Namespace) -> int: ) print( f"Snapshot SHA-256: {snapshot_hash}; " - f"Reference rows: {len(references.references)}" + f"Human findings: {len(human_audit.human_findings)}" ) print(f"Pricing: {schedule.version} ({schedule.identity})") print(f"Cost guardrail: ${limit}; checked before each request") @@ -190,24 +192,24 @@ def _report(args: argparse.Namespace) -> int: raise ValueError("Run evidence does not belong to this directory") if not manifest["complete"]: raise ValueError("Incomplete Evaluation runs cannot produce a final report") - reference_set = json.loads((run / "references.json").read_text()) - if reference_set["workbook_sha256"] != manifest["workbook_sha256"]: - raise ValueError("Reference workbook identity differs from the run") - audit = json.loads((run / "normalized-audit-findings.json").read_text()) + human_audit = json.loads((run / "human-findings.json").read_text()) + if human_audit["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the run") + llm = json.loads((run / "normalized-llm-findings.json").read_text()) programmatic_file = run / "normalized-programmatic-findings.json" programmatic = json.loads(programmatic_file.read_text()) - for kind, collection in (("audit", audit), ("programmatic", programmatic)): + for kind, collection in (("llm", llm), ("programmatic", programmatic)): if any( item["run_id"] != run.name or item["source"] != kind or item["page_url"] != manifest["source_url"] for item in collection ): - raise ValueError(f"{kind} findings do not belong to this run") + raise ValueError(f"{kind.upper() if kind == 'llm' else kind.title()} findings do not belong to this run") rows = reviewed_rows( - run / "review.csv", reference_set["references"], audit, programmatic + run / "review.csv", human_audit["human_findings"], llm, programmatic ) - write_report(run / "report.md", manifest, rows, audit, programmatic) + write_report(run / "report.md", manifest, rows, llm, programmatic) print(f"Markdown report: {run / 'report.md'}") return 0 diff --git a/vision_aid/evaluation/references.py b/vision_aid/evaluation/human_audit.py similarity index 68% rename from vision_aid/evaluation/references.py rename to vision_aid/evaluation/human_audit.py index fbfc026..05793ed 100644 --- a/vision_aid/evaluation/references.py +++ b/vision_aid/evaluation/human_audit.py @@ -1,4 +1,4 @@ -"""Import the known Reference workbook's Home and Global defect rows.""" +"""Import the known Human audit's Home and Global defect rows.""" from __future__ import annotations @@ -8,7 +8,7 @@ import openpyxl -from vision_aid.evaluation.schemas import ReferenceDefect, ReferenceSet +from vision_aid.evaluation.schemas import HumanAudit, HumanFinding SHEET = "Defect report" HEADERS = ( @@ -33,17 +33,17 @@ ) -def import_homepage_references(workbook_path: Path, homepage_url: str) -> ReferenceSet: +def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: """Preserve every applicable source row and its original cell text.""" - checksum = hashlib.sha256(workbook_path.read_bytes()).hexdigest() - book = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + book = openpyxl.load_workbook(path, read_only=True, data_only=True) try: sheet = book[SHEET] rows = sheet.iter_rows(values_only=True) headers = tuple(str(value or "") for value in next(rows)) if headers[: len(HEADERS)] != HEADERS: - raise ValueError("Reference workbook has unexpected Defect report columns") - references = [] + raise ValueError("Human audit has unexpected Defect report columns") + human_findings = [] for number, values in enumerate(rows, start=2): raw = { header: "" if value is None else str(value) @@ -56,9 +56,9 @@ def import_homepage_references(workbook_path: Path, homepage_url: str) -> Refere if not raw["Issue Title"].strip(): raise ValueError(f"Defect report row {number} has no Issue Title") identity = f"{checksum}:{SHEET}:{number}".encode() - references.append( - ReferenceDefect( - reference_id="ref-" + hashlib.sha256(identity).hexdigest()[:16], + human_findings.append( + HumanFinding( + human_finding_id="human-" + hashlib.sha256(identity).hexdigest()[:16], source_sheet=SHEET, source_row=number, page_scope=scope, @@ -72,6 +72,6 @@ def import_homepage_references(workbook_path: Path, homepage_url: str) -> Refere ) finally: book.close() - if not references: - raise ValueError("Reference workbook has no Home or Global defects") - return ReferenceSet(workbook_path.name, checksum, homepage_url, tuple(references)) + if not human_findings: + raise ValueError("Human audit has no Home or Global defects") + return HumanAudit(path.name, checksum, homepage_url, tuple(human_findings)) diff --git a/vision_aid/evaluation/audit.py b/vision_aid/evaluation/llm_audit.py similarity index 92% rename from vision_aid/evaluation/audit.py rename to vision_aid/evaluation/llm_audit.py index d37d2f8..3645e66 100644 --- a/vision_aid/evaluation/audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -8,12 +8,15 @@ from pathlib import Path from typing import Any -from processing_scripts.llm_client.audit import AuditRequestClient, AuditRequestConfig +from processing_scripts.llm_client.audit import ( + AuditRequestClient as LLMRequestClient, + AuditRequestConfig as LLMRequestConfig, +) from vision_aid.evaluation.normalization import normalize_prompt_response from vision_aid.evaluation.pricing import PriceSchedule MODEL = "claude-haiku-4-5-20251001" -REQUEST_CONFIG = AuditRequestConfig( +REQUEST_CONFIG = LLMRequestConfig( model=MODEL, thinking_budget_tokens=16000, max_output_tokens=24192, @@ -58,7 +61,7 @@ def positive_cost(value: str) -> Decimal: def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: """Make each paid request once, saving reported usage and responses as received.""" schedule = PriceSchedule.load(run / "pricing.json") - client = AuditRequestClient( + client = LLMRequestClient( api_key=api_key, request_config=REQUEST_CONFIG, max_retries=0, @@ -114,8 +117,8 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: incomplete_reason=reason, format_failures=format_failures, ) - write_json(run / "raw-audit-responses.json", responses) - write_json(run / "normalized-audit-findings.json", findings) + write_json(run / "raw-llm-responses.json", responses) + write_json(run / "normalized-llm-findings.json", findings) write_json(run / "run.json", manifest) if not complete: break diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index a5fb14f..88a170a 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -1,4 +1,4 @@ -"""Lossless deterministic normalization of raw audit outcomes.""" +"""Lossless deterministic normalization of raw LLM outcomes.""" from __future__ import annotations @@ -225,8 +225,8 @@ def normalize_prompt_response( identity = f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() findings.append( CanonicalFinding( - finding_id="finding-" + hashlib.sha256(identity).hexdigest()[:20], - source="audit", + finding_id="llm-" + hashlib.sha256(identity).hexdigest()[:20], + source="llm", run_id=run_id, model=model, prompt=prompt_name, diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index f66e48b..3219a4a 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -8,18 +8,18 @@ from pathlib import Path from typing import Any -from vision_aid.evaluation.references import ELIGIBILITY_VALUES, HEADERS +from vision_aid.evaluation.human_audit import ELIGIBILITY_VALUES, HEADERS DECISION_COLUMNS = ( "classification", "classification_reason", - "audit_finding_id", + "llm_finding_id", "programmatic_finding_id", "match_reason", "review_notes", ) REVIEW_COLUMNS = ( - "reference_id", + "human_finding_id", "source_sheet", "source_row", "page_scope", @@ -29,15 +29,15 @@ ) -def create_review(path: Path, references: list[dict[str, Any]]) -> None: - """Create one editable row per Reference defect with blank decisions.""" +def create_review(path: Path, human_findings: list[dict[str, Any]]) -> None: + """Create one editable row per Human finding with blank decisions.""" with path.open("w", encoding="utf-8", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=REVIEW_COLUMNS) writer.writeheader() - for item in references: + for item in human_findings: writer.writerow( { - "reference_id": item["reference_id"], + "human_finding_id": item["human_finding_id"], "source_sheet": item["source_sheet"], "source_row": item["source_row"], "page_scope": item["page_scope"], @@ -58,8 +58,8 @@ def _ids(value: str) -> tuple[str, ...]: def reviewed_rows( path: Path, - references: list[dict[str, Any]], - audit: list[dict[str, Any]], + human_findings: list[dict[str, Any]], + llm: list[dict[str, Any]], programmatic: list[dict[str, Any]], ) -> list[dict[str, Any]]: """Validate completeness and finding kind before granting any row credit.""" @@ -68,57 +68,57 @@ def reviewed_rows( if not set(REVIEW_COLUMNS).issubset(reader.fieldnames or ()): raise ValueError("Review CSV is missing required columns") entered = list(reader) - expected = {item["reference_id"]: item for item in references} + expected = {item["human_finding_id"]: item for item in human_findings} if len(entered) != len(expected): - raise ValueError("Review CSV must contain each Reference defect exactly once") + raise ValueError("Review CSV must contain each Human finding exactly once") by_kind = { - "audit": {item["finding_id"]: item for item in audit}, + "llm": {item["finding_id"]: item for item in llm}, "programmatic": {item["finding_id"]: item for item in programmatic}, } seen_rows: set[str] = set() - used: dict[str, set[str]] = {"audit": set(), "programmatic": set()} + used: dict[str, set[str]] = {"llm": set(), "programmatic": set()} result = [] for row in entered: - reference_id = row["reference_id"] - if reference_id not in expected or reference_id in seen_rows: - raise ValueError(f"Unknown or repeated Reference defect: {reference_id}") - seen_rows.add(reference_id) - reference = expected[reference_id] + human_finding_id = row["human_finding_id"] + if human_finding_id not in expected or human_finding_id in seen_rows: + raise ValueError(f"Unknown or repeated Human finding: {human_finding_id}") + seen_rows.add(human_finding_id) + human_finding = expected[human_finding_id] for field in ("source_sheet", "source_row", "source_element"): - if str(row[field]) != str(reference[field]): + if str(row[field]) != str(human_finding[field]): raise ValueError( - f"Reference identity changed for {reference_id}: {field}" + f"Human finding identity changed for {human_finding_id}: {field}" ) classification = row["classification"].strip() if classification not in ELIGIBILITY_VALUES: - raise ValueError(f"Reference {reference_id} needs a valid classification") + raise ValueError(f"Human finding {human_finding_id}: invalid classification") if not row["classification_reason"].strip(): - raise ValueError(f"Reference {reference_id} needs a classification reason") + raise ValueError(f"Human finding {human_finding_id}: missing classification reason") matched: dict[str, tuple[str, ...]] = {} - for kind, column in ( - ("audit", "audit_finding_id"), - ("programmatic", "programmatic_finding_id"), + for kind, column, label in ( + ("llm", "llm_finding_id", "LLM"), + ("programmatic", "programmatic_finding_id", "Programmatic"), ): ids = _ids(row[column]) unknown = set(ids) - set(by_kind[kind]) if unknown: raise ValueError( - f"Unknown {kind} finding ID: {', '.join(sorted(unknown))}" + f"Unknown {label} finding ID: {', '.join(sorted(unknown))}" ) reused = set(ids) & used[kind] if reused: raise ValueError( - f"Conflicting {kind} finding ID: {', '.join(sorted(reused))}" + f"Conflicting {label} finding ID: {', '.join(sorted(reused))}" ) used[kind].update(ids) matched[kind] = ids if any(matched.values()) and not row["match_reason"].strip(): - raise ValueError(f"Reference {reference_id} needs a match reason") + raise ValueError(f"Human finding {human_finding_id} needs a match reason") if classification in {"unavailable_evidence", "ambiguous"} and any( matched.values() ): - raise ValueError(f"Excluded Reference {reference_id} cannot claim coverage") - result.append({"reference": reference, "decision": row, "matches": matched}) + raise ValueError(f"Human finding {human_finding_id} is not testable; cannot claim coverage") + result.append({"human_finding": human_finding, "decision": row, "matches": matched}) return result @@ -156,19 +156,19 @@ def write_report( path: Path, manifest: dict[str, Any], rows: list[dict[str, Any]], - audit: list[dict[str, Any]], + llm: list[dict[str, Any]], programmatic: list[dict[str, Any]], ) -> None: """Write the single human-readable score and row evidence projection.""" if not manifest["complete"]: raise ValueError("Incomplete Evaluation runs cannot produce a final report") - findings = {item["finding_id"]: item for item in audit + programmatic} - llm = [row for row in rows if row["decision"]["classification"] == "llm_eligible"] + findings = {item["finding_id"]: item for item in llm + programmatic} + llm_rows = [row for row in rows if row["decision"]["classification"] == "llm_eligible"] checks = [ row for row in rows if row["decision"]["classification"] == "programmatic" ] covered = [row for row in rows if any(row["matches"].values())] - audit_caught = sum(bool(row["matches"]["audit"]) for row in llm) + llm_caught = sum(bool(row["matches"]["llm"]) for row in llm_rows) programmatic_caught = sum(bool(row["matches"]["programmatic"]) for row in checks) unavailable = sum( row["decision"]["classification"] == "unavailable_evidence" for row in rows @@ -178,15 +178,15 @@ def write_report( f"# Evaluation run {manifest['run_id']}", "", f"- Model: {manifest['configuration']['model']}", - f"- Workbook-row recall: {_metric(audit_caught, len(llm))}", - f"- Programmatic coverage: {_metric(programmatic_caught, len(checks))}", - f"- Combined workbook coverage: {_metric(len(covered), len(rows))}", + f"- LLM detection rate: {_metric(llm_caught, len(llm_rows))}", + f"- Programmatic detection rate: {_metric(programmatic_caught, len(checks))}", + f"- Overall detection rate: {_metric(len(covered), len(rows))}", f"- Estimated run cost: ${manifest['estimated_cost_usd']}", f"- Unavailable evidence: {unavailable}; ambiguous: {ambiguous}", f"- Reported usage: {_md(json.dumps(manifest['usage'], sort_keys=True))}", "", ( - "This is full-row coverage of this homepage workbook only. " + "This is full-row coverage of this homepage Human audit only. " "A blank finding cell means missed." ), ( @@ -198,27 +198,27 @@ def write_report( f"{_md(', '.join(manifest['format_failures']) or 'none')}" ), "", - "## Reference defects", + "## Human findings", "", ] for row in rows: - reference = row["reference"] + human_finding = row["human_finding"] decision = row["decision"] matches = row["matches"] lines.extend( [ - f"### {reference['reference_id']} — {_md(reference['problem'])}", + f"### {human_finding['human_finding_id']} — {_md(human_finding['problem'])}", "", ( - f"- Source: {reference['source_sheet']} " - f"row {reference['source_row']} ({reference['page_scope']})" + f"- Source: {human_finding['source_sheet']} " + f"row {human_finding['source_row']} ({human_finding['page_scope']})" ), - f"- source_element: {_md(reference['source_element'])}", + f"- source_element: {_md(human_finding['source_element'])}", ( f"- Classification: {decision['classification']} — " f"{_md(decision['classification_reason'])}" ), - f"- Audit findings: {_md('; '.join(matches['audit']) or 'none')}", + f"- LLM findings: {_md('; '.join(matches['llm']) or 'none')}", ( "- Programmatic findings: " f"{_md('; '.join(matches['programmatic']) or 'none')}" @@ -227,11 +227,11 @@ def write_report( f"- Review notes: {_md(decision['review_notes'] or 'none')}", ] ) - for finding_id in (*matches["audit"], *matches["programmatic"]): + for finding_id in (*matches["llm"], *matches["programmatic"]): finding = findings[finding_id] lines.append( f"- {finding_id}: {_md(finding['problem'])} " f"@ {_md(finding['location'])}" ) - lines.extend(["", *_evidence_block(reference["raw_evidence"]), ""]) + lines.extend(["", *_evidence_block(human_finding["raw_evidence"]), ""]) path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index cafd500..facd746 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -7,8 +7,8 @@ @dataclass(frozen=True) -class ReferenceDefect: - reference_id: str +class HumanFinding: + human_finding_id: str source_sheet: str source_row: int page_scope: Literal["home", "global"] @@ -19,11 +19,11 @@ class ReferenceDefect: @dataclass(frozen=True) -class ReferenceSet: - workbook_filename: str - workbook_sha256: str +class HumanAudit: + human_audit_filename: str + human_audit_sha256: str homepage_url: str - references: tuple[ReferenceDefect, ...] + human_findings: tuple[HumanFinding, ...] def to_dict(self) -> dict[str, Any]: """Return JSON-compatible source evidence.""" @@ -33,7 +33,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class CanonicalFinding: finding_id: str - source: Literal["audit", "programmatic"] + source: Literal["llm", "programmatic"] run_id: str model: str | None prompt: str From 61ac26c01fefdfc9abd5bf5ad19f981a728acfd9 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 17:34:16 -0400 Subject: [PATCH 047/126] chore(tickets): resolve Human audit vocabulary ticket --- ...ename-to-human-audit-and-llm-vocabulary.md | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md b/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md index 48508d5..e43005f 100644 --- a/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md +++ b/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md @@ -4,20 +4,30 @@ **Blocked by:** None (can start immediately). -**Status:** ready-for-agent - -- [ ] The `run` subcommand accepts `--human-audit` in place of `--workbook`, with the default-path constant renamed to match -- [ ] A run writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`, and never writes `references.json`, `raw-audit-responses.json` or `normalized-audit-findings.json` (asserted in tests) -- [ ] `review.csv` uses `human_finding_id` and `llm_finding_id` columns, and `report` validates against those columns -- [ ] Human finding IDs start with `human-`, LLM finding IDs with `llm-`, and Programmatic finding IDs with `programmatic-`; the hash part of each ID is unchanged -- [ ] LLM findings record `source` as `"llm"` -- [ ] The import module/function and records are renamed (`human_audit`, `import_homepage_human_findings`, `HumanFinding`, `HumanAudit` with `human_findings`) -- [ ] The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match -- [ ] The manifest records `human_audit_filename` and `human_audit_sha256`, and `report` checks the Human audit checksum against them -- [ ] CLI help and error messages say "Human audit", "Human finding" and "LLM finding" instead of "Reference workbook", "Reference defect", "Excluded Reference" or "audit finding" -- [ ] The README's evaluation workflow, run-directory file table, review-column guide and troubleshooting text use the new names -- [ ] Classification values, the spreadsheet's sheet and column headers, and the programmatic artifact names are unchanged -- [ ] Evaluation CLI tests pass through the `main` seam with the fake LLM client, and the full test suite passes -- [ ] The commit is marked breaking (`!` and a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported) +**Status:** resolved + +- [x] The `run` subcommand accepts `--human-audit` in place of `--workbook`, with the default-path constant renamed to match +- [x] A run writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`, and never writes `references.json`, `raw-audit-responses.json` or `normalized-audit-findings.json` (asserted in tests) +- [x] `review.csv` uses `human_finding_id` and `llm_finding_id` columns, and `report` validates against those columns +- [x] Human finding IDs start with `human-`, LLM finding IDs with `llm-`, and Programmatic finding IDs with `programmatic-`; the hash part of each ID is unchanged +- [x] LLM findings record `source` as `"llm"` +- [x] The import module/function and records are renamed (`human_audit`, `import_homepage_human_findings`, `HumanFinding`, `HumanAudit` with `human_findings`) +- [x] The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match +- [x] The manifest records `human_audit_filename` and `human_audit_sha256`, and `report` checks the Human audit checksum against them +- [x] CLI help and error messages say "Human audit", "Human finding" and "LLM finding" instead of "Reference workbook", "Reference defect", "Excluded Reference" or "audit finding" +- [x] The README's evaluation workflow, run-directory file table, review-column guide and troubleshooting text use the new names +- [x] Classification values, the spreadsheet's sheet and column headers, and the programmatic artifact names are unchanged +- [x] Evaluation CLI tests pass through the `main` seam with the fake LLM client, and the full test suite passes +- [x] The commit is marked breaking (`!` and a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported) ## Comments + +### Completed 2026-09-29 + +Implementation: `401822c9a4a60ac4e196f0bef53e07432990aed5` (`refactor(evaluation)!: adopt Human audit and LLM vocabulary`). + +Verification: `uv run pytest -q tests/test_evaluation_cli.py` passed all 8 cases; `uv run pytest -q` passed all 37 tests. `uv run python -m compileall -q vision_aid/evaluation tests/test_evaluation_cli.py` and `git diff --check` passed. No dedicated typechecker is configured. Existing CLI main-seam cases verify the new Human audit flag, artifact names and legacy artifact absence, CSV columns, finding prefixes/source values, manifest provenance, legacy flag/CSV rejection, and Human audit checksum mismatch rejection using fake LLM clients. No paid requests were made. + +Review: independent Standards and Spec agents reviewed the diff from `12c17ab311ae30e62e208f08784760c5a4449c12`; both reported zero findings. Spec review confirmed unchanged ID hash construction, classification values, source sheet/headers, Programmatic artifact names, report layout, scoring, and shared pipeline. Shared request classes are imported under LLM names only within the evaluator. The implementation commit carries `!` and the required `BREAKING CHANGE:` footer. + +Size: production Python additions against main are 1,567 versus 1,562 before this ticket; all Python including tests is 2,565 versus 2,534. From be15c06e50b07e4033e278c4ac79cfd8248c51ba Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 17:42:30 -0400 Subject: [PATCH 048/126] feat(evaluation)!: explain catches and misses in plain language Require reviewer attribution, retain detection-rate scoring, and allow empty classification reasons as directed by the operator. BREAKING CHANGE: Existing Evaluation run directories are unsupported; report requires --reviewer. --- README.md | 26 ++-- tests/test_evaluation_cli.py | 224 +++++++++++++++++--------------- vision_aid/evaluation/cli.py | 9 +- vision_aid/evaluation/review.py | 203 +++++++++++++++-------------- 4 files changed, 249 insertions(+), 213 deletions(-) diff --git a/README.md b/README.md index af8e8ce..15684fa 100644 --- a/README.md +++ b/README.md @@ -297,16 +297,24 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report Excel or a CSV-aware text editor. Review every Human finding and fill in the decision columns using the guide below. Save the file as CSV. -6. **Generate the final report.** Run `report` after human review: +6. **Generate the final report.** Run `report` after completing the review, + naming whoever made the decisions: ```bash - uv run visionaid-evaluate report --run-dir "$RUN_DIR" + uv run visionaid-evaluate report --run-dir "$RUN_DIR" --reviewer "Your name or agent description" ``` - Open `$RUN_DIR/report.md` for LLM detection rate, Programmatic detection rate, - Overall detection rate, Estimated run cost, limitations, and source-row - explanations. Existing Evaluation run directories are unsupported. Reporting - rejects incomplete runs, missing decisions, and invalid or reused finding IDs. + Open `$RUN_DIR/report.md`. It explains how to read the homepage evaluation, + then shows the model, run ID, reviewer, three detection rates, and estimated + cost with token counts and the pricing date. A table follows the Human audit's + original order and marks catches by each part, including cross-catches. The + remaining sections show what each part caught and why each match counts, + every missed finding with any recorded explanations, and findings not + testable from the page file with their reasons. IDs, locations, and raw JSON + stay in the run artifacts. Not-testable findings are left out of the LLM and + Programmatic rates but remain in the Overall denominator. Existing Evaluation + run directories are unsupported. Reporting rejects incomplete runs, missing + classifications, and invalid or reused finding IDs. ### Evaluation run files @@ -354,7 +362,7 @@ In particular, `source_sheet` and `source_row` point back to the Human audit, `page_scope` says Home or Global, and `source_element` copies the Human audit's `element name`. It is not an inferred location on the page. -For **every row**, fill in `classification` and `classification_reason`: +For **every row**, fill in `classification`: | `classification` value | Use it when | | --- | --- | @@ -366,8 +374,8 @@ For **every row**, fill in `classification` and `classification_reason`: Base the classification on the Human finding and available evidence, not on whether this run happened to produce a matching finding. In `classification_reason`, write a short explanation of that choice, including -what evidence is missing or unclear when applicable. These two cells cannot be -blank when you generate the report. +what evidence is missing or unclear when applicable. This explanation is +optional, but `classification` cannot be blank when you generate the report. Then complete the remaining decision columns for that row: diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index a92a6df..42dd764 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -33,54 +33,15 @@ def inputs(tmp_path: Path) -> tuple[Path, Path]: "comment", ] ) - sheet.append( - [ - 1, - "First link", - "Chrome", - "Home", - "Unclear link", - "Open page", - "Vague link\nfor everyone", - "Descriptive text", - "Rename", - "2.4.4", - "Code", - "", - ] - ) - sheet.append( - [ - 2, - "Second link", - "Chrome", - "Global", - "Missing destination", - "Open page", - "No href", - "Working link", - "Add href", - "2.4.4", - "Code", - "", - ] - ) - sheet.append( - [ - 3, - "Photo", - "Chrome", - "Home", - "No alt text", - "Open page", - "Image lacks alt", - "Alt text", - "Add alt", - "1.1.1", - "Code", - "", - ] - ) + for row in ( + [1, "First link", "Chrome", "Home", "Unclear link", "Open page", + "Vague link\nfor everyone", "Descriptive text", "Rename", "2.4.4", "Code", ""], + [2, "Second link", "Chrome", "Global", "Missing destination", "Open page", + "No href", "Working link", "Add href", "2.4.4", "Code", ""], + [3, "Photo", "Chrome", "Home", "No alt text", "Open page", + "Image lacks alt", "Alt text", "Add alt", "1.1.1", "Code", ""], + ): + sheet.append(row) sheet.append([4, "Other", "Chrome", "Careers", "Outside scope"]) human_audit = tmp_path / "human-audit.xlsx" book.save(human_audit) @@ -169,7 +130,7 @@ def test_preview_preserves_human_findings_and_never_calls_provider( with pytest.raises(SystemExit): main([*args(human_audit, html), "--workbook", str(human_audit)]) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) def test_subdirectory_invocation_uses_ignored_worktree_run_root( @@ -206,10 +167,10 @@ def call(self, prompt: str) -> dict: [{"problem": "Unclear link", "location": "First link"}] ), "usage": { - "input_tokens": 100, - "output_tokens": 20, - "cached_input_tokens": 50, - "cache_creation_input_tokens": 10, + "input_tokens": 1000, + "output_tokens": 200, + "cached_input_tokens": 500, + "cache_creation_input_tokens": 100, "reasoning_tokens": None, }, } @@ -228,7 +189,7 @@ def client(**options: object) -> FakeClient: run = directory(tmp_path) assert fake.calls == 3 assert ( - json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.0006525" + json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.006525" ) assert (run / "raw-llm-responses.json").exists() assert all(not (run / name).exists() for name in ( @@ -245,7 +206,7 @@ def client(**options: object) -> FakeClient: rows[0].update( classification="llm_eligible", classification_reason="Model can see link", - llm_finding_id=findings[0]["finding_id"], + llm_finding_id=findings[-1]["finding_id"], match_reason="Full issue", ) rows[1].update( @@ -258,79 +219,120 @@ def client(**options: object) -> FakeClient: classification="unavailable_evidence", classification_reason="Visual context" ) write_review(review, rows) - assert main(["report", "--run-dir", str(run)]) == 0 + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run)]) + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run), "--reviewer", " "]) + write_review(review, list(reversed(rows))) + assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 report = (run / "report.md").read_text() - assert "LLM detection rate: 1/1" in report - assert "Programmatic detection rate: 1/1" in report - assert "Overall detection rate: 2/3" in report - assert "Vague link" in report and "First link" in report + assert "# LLM accessibility evaluation: https://example.test/" in report + headings = ["How to read this report", "Summary", "Human findings", "What each part of the tool caught", "Missed", "Not testable from the page file"] + assert [report.index("## " + heading + "\n") for heading in headings] == sorted(report.index("## " + heading + "\n") for heading in headings) + assert all(text in report for text in ( + "benchmark is the Human audit", "The tool has two parts", "large language model", + "fixed rules without AI", "only saved HTML", "homepage only", "A reviewer decided", + "fully cover the whole Human finding", "Partial coverage earns no credit", + )) + assert "Review decisions by: Codex agent (operator-directed)" in report + assert "claude-haiku-4-5-20251001" in report and run.name in report + assert "Of the 1 Human findings the LLM could be expected to detect from the page's HTML, it caught 1 (100.0%)" in report + assert "Programmatic detection rate: Of the 1 Human findings the Programmatic checks could be expected to detect from the page's HTML, they caught 1 (100.0%)" in report + assert "Of all 3 Human findings, either part of the tool caught 2 (66.7%)" in report + assert "Estimated run cost: $0.01" in report + assert "3000 input tokens" in report and "600 output tokens" in report + assert "1500 cached input tokens" in report and "300 cache-creation input tokens" in report + assert "2026-09-22" in report and "The actual bill may differ slightly." in report + assert "unreadable output" not in report + table = report.split("## Human findings\n", 1)[1].split("## What each part", 1)[0] + assert table.index("Unclear link") < table.index("Missing destination") < table.index("No alt text") + assert "| Unclear link | 2.4.4 | LLM | ✓ | ✗ |" in table + assert "| Missing destination | 2.4.4 | Programmatic checks | ✗ | ✓ |" in table + assert "LLM (link clarity check): Unclear link" in report + assert "Programmatic check " + programmatic[0]["prompt"].replace("_", r"\_") in report + assert "Why this counts: Full issue" in report + assert "Visual context" in report + assert all(row["human_finding_id"] not in report for row in rows) + assert all(item["finding_id"] not in report for item in findings + programmatic) + assert not any(text in report for text in ("source_element", "First link", "```", "
", "reasoning_tokens", "@ ")) for row in rows: row["audit_finding_id"] = row.pop("llm_finding_id") write_review(review, rows) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) imported = json.loads((run / "human-findings.json").read_text()) imported["human_audit_sha256"] = "changed" (run / "human-findings.json").write_text(json.dumps(imported)) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) -def test_report_shows_evidence_verbatim_and_escapes_inline_html( +def test_report_cross_catches_empty_misses_and_not_testable_findings( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Fenced evidence stays verbatim and inline fields escape raw HTML.""" + """Cross-catches count overall, blank misses survive, and text is escaped.""" human_audit, html = inputs(tmp_path) book = openpyxl.load_workbook(human_audit) - actual = 'Use
tag.\nSay "Main" ```here```' - book.active["G2"] = actual + book.active["E2"] = "Unclear
|link" + book.active.append([5, "Menu", "Chrome", "Home", "Hidden menu", "", "", "", "", "2.1.1"]) + book.active.append([6, "Widget", "Chrome", "Home", "Unclear claim", "", "", "", "", "4.1.2"]) book.save(human_audit) monkeypatch.chdir(tmp_path) monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") class FakeClient: + calls = 0 + def call(self, prompt: str) -> dict: - """Return one normalized finding for applicable prompts.""" + """One unreadable response and two findings with private locations.""" + self.calls += 1 return { "success": True, - "response": json.dumps([{"problem": "Vague link"}]), + "response": "not JSON" if self.calls == 1 else json.dumps([{"problem": "Use
|landmark", "location": "body > main"}]), "usage": {"input_tokens": 1, "output_tokens": 1}, } - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) + fake = FakeClient() + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: fake) assert main(args(human_audit, html, "--live", "--approve-live")) == 0 run = directory(tmp_path) + findings = json.loads((run / "normalized-llm-findings.json").read_text()) review = run / "review.csv" rows = read_review(review) - for row in rows: - row.update(classification="ambiguous", classification_reason="Unclear") - rows[0]["classification_reason"] = 'An needs judgment' - write_review(review, rows) - assert main(["report", "--run-dir", str(run)]) == 0 + rows[0].update(classification="llm_eligible", llm_finding_id=findings[0]["finding_id"], match_reason="Full
|coverage") + rows[1].update(classification="programmatic", llm_finding_id=findings[1]["finding_id"], match_reason="Cross-catch") + rows[2].update(classification="programmatic") + rows[3].update(classification="unavailable_evidence", classification_reason="Requires a live browser") + rows[4].update(classification="ambiguous", classification_reason='An |needs judgment') + write_review(review, list(reversed(rows))) + assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 report = (run / "report.md").read_text() - evidence = ( - "
Original defect evidence\n\n" - "````json\n" - "{\n" - ' "Browser Combination": "Chrome",\n' - ' "Issue Title": "Unclear link",\n' - ' "Page name": "Home",\n' - ' "Recommendation for Fix": "Rename",\n' - ' "Sr. #": "1",\n' - ' "Steps to Reproduce": "Open page",\n' - ' "Type of Change": "Code",\n' - ' "WCAG Sc": "2.4.4",\n' - ' "actual result": "Use
tag.\\nSay \\"Main\\" ```here```",\n' - ' "comment": "",\n' - ' "element name": "First link",\n' - ' "expected result": "Descriptive text"\n' - "}\n" - "````\n\n" - "
\n" - ) - assert evidence in report - assert "Original defect evidence:" not in report - assert 'An \\ needs judgment' in report + assert r"Unclear \\|link" in report + assert r"Use \\|landmark" in report + assert r"Why this counts: Full \\|coverage" in report + assert "| Missing destination | 2.4.4 | Programmatic checks | ✓ | ✗ |" in report + assert "it caught 1 (100.0%)" in report and "they caught 0 (0.0%)" in report + assert "Of all 5 Human findings, either part of the tool caught 2 (40.0%)" in report + assert "count only catches by the expected part" in report + missed = report.split("## Missed\n", 1)[1].split("## Not testable", 1)[0] + assert "### No alt text" in missed + assert "Why it was expected to be caught" not in missed and "Why it was missed" not in missed + excluded = report.split("## Not testable from the page file\n", 1)[1] + assert "### Hidden menu" in excluded and "Requires a live browser" in excluded + assert "### Unclear claim" in excluded and r'An \\|needs judgment' in excluded + assert "remain in the Overall denominator" in excluded + assert "3 input tokens" in report and "3 output tokens" in report + assert "cached input" not in report and "cache-creation" not in report + assert "The LLM returned unreadable output for 1 of 3 checks: page title check." in report + assert all(row["human_finding_id"] not in report for row in rows) + assert all(item["finding_id"] not in report for item in findings) + assert not any(text in report for text in ("
", '', "body > main", "```", "source_element")) + rows[2].update(classification_reason="The image is in the HTML", review_notes="No full finding") + write_review(review, rows) + assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 + explained = (run / "report.md").read_text().split("## Missed\n", 1)[1] + assert "Why it was expected to be caught: The image is in the HTML" in explained + assert "Why it was missed: No full finding" in explained def test_review_requires_classification_and_rejects_invalid_finding_ids( @@ -365,7 +367,7 @@ def call(self, prompt: str) -> dict: )[0]["finding_id"] rows = read_review(review) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) rows[0].update( classification="llm_eligible", classification_reason="Visible", @@ -377,23 +379,29 @@ def call(self, prompt: str) -> dict: rows[1].update(classification="programmatic", classification_reason="Markup") rows[2].update(classification="ambiguous", classification_reason="Unclear claim") write_review(review, rows) - assert main(["report", "--run-dir", str(run)]) == 0 - assert "Overall detection rate: 1/3" in (run / "report.md").read_text() + assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 + assert "Of all 3 Human findings, either part of the tool caught 1 (33.3%)" in (run / "report.md").read_text() + rows[0]["match_reason"] = "" + write_review(review, rows) + with pytest.raises(SystemExit): + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) + assert "needs a match reason" in capsys.readouterr().err + rows[0]["match_reason"] = "Together cover the row" rows[1]["llm_finding_id"] = llm_ids[0] rows[1]["match_reason"] = "Conflicts" write_review(review, rows) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) assert "Conflicting LLM finding ID" in capsys.readouterr().err rows[1]["llm_finding_id"] = "llm-unknown" write_review(review, rows) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) assert "Unknown LLM finding ID" in capsys.readouterr().err rows[1]["llm_finding_id"] = programmatic_id write_review(review, rows) with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) assert "Unknown LLM finding ID" in capsys.readouterr().err rows[0]["llm_finding_id"] = "" rows[0]["programmatic_finding_id"] = "" @@ -401,13 +409,15 @@ def call(self, prompt: str) -> dict: rows[0]["classification"] = "ambiguous" rows[1]["llm_finding_id"] = "" rows[1]["match_reason"] = "" + rows[1]["classification"] = "ambiguous" rows[2]["classification"] = "unavailable_evidence" write_review(review, rows) - assert main(["report", "--run-dir", str(run)]) == 0 + assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 assert ( - "LLM detection rate: N/A (0 eligible rows)" in (run / "report.md").read_text() + "No Human findings were expected to be detected by the LLM" in (run / "report.md").read_text() ) - assert "Overall detection rate: 0/3" in (run / "report.md").read_text() + assert "No Human findings were expected to be detected by the Programmatic checks" in (run / "report.md").read_text() + assert "Of all 3 Human findings, either part of the tool caught 0 (0.0%)" in (run / "report.md").read_text() def test_live_execution_needs_explicit_approval( @@ -466,4 +476,4 @@ def call(self, prompt: str) -> dict: assert float(manifest["estimated_cost_usd"]) > 0 assert len(json.loads((run / "raw-llm-responses.json").read_text())) == 1 with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) + main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 68981b3..c52d236 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -59,6 +59,7 @@ def build_parser() -> argparse.ArgumentParser: "report", help="score completed run and edited review.csv" ) report.add_argument("--run-dir", type=Path, required=True) + report.add_argument("--reviewer", required=True, help="who made the review decisions") return parser @@ -186,6 +187,8 @@ def _run(args: argparse.Namespace) -> int: def _report(args: argparse.Namespace) -> int: """Validate human decisions and report a completed run.""" + if not args.reviewer.strip(): + raise ValueError("A reviewer description is required") run = args.run_dir.resolve() manifest = json.loads((run / "run.json").read_text()) if manifest["run_id"] != run.name: @@ -209,7 +212,11 @@ def _report(args: argparse.Namespace) -> int: rows = reviewed_rows( run / "review.csv", human_audit["human_findings"], llm, programmatic ) - write_report(run / "report.md", manifest, rows, llm, programmatic) + schedule = PriceSchedule.load(run / "pricing.json") + write_report( + run / "report.md", manifest, rows, llm, programmatic, + args.reviewer, str(schedule.data["published_at"]), + ) print(f"Markdown report: {run / 'report.md'}") return 0 diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 3219a4a..3d7a55f 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -3,8 +3,7 @@ from __future__ import annotations import csv -import json -import re +from decimal import Decimal from pathlib import Path from typing import Any @@ -75,14 +74,12 @@ def reviewed_rows( "llm": {item["finding_id"]: item for item in llm}, "programmatic": {item["finding_id"]: item for item in programmatic}, } - seen_rows: set[str] = set() used: dict[str, set[str]] = {"llm": set(), "programmatic": set()} - result = [] + result: dict[str, dict[str, Any]] = {} for row in entered: human_finding_id = row["human_finding_id"] - if human_finding_id not in expected or human_finding_id in seen_rows: + if human_finding_id not in expected or human_finding_id in result: raise ValueError(f"Unknown or repeated Human finding: {human_finding_id}") - seen_rows.add(human_finding_id) human_finding = expected[human_finding_id] for field in ("source_sheet", "source_row", "source_element"): if str(row[field]) != str(human_finding[field]): @@ -92,8 +89,6 @@ def reviewed_rows( classification = row["classification"].strip() if classification not in ELIGIBILITY_VALUES: raise ValueError(f"Human finding {human_finding_id}: invalid classification") - if not row["classification_reason"].strip(): - raise ValueError(f"Human finding {human_finding_id}: missing classification reason") matched: dict[str, tuple[str, ...]] = {} for kind, column, label in ( ("llm", "llm_finding_id", "LLM"), @@ -118,120 +113,136 @@ def reviewed_rows( matched.values() ): raise ValueError(f"Human finding {human_finding_id} is not testable; cannot claim coverage") - result.append({"human_finding": human_finding, "decision": row, "matches": matched}) - return result + result[human_finding_id] = { + "human_finding": human_finding, "decision": row, "matches": matched, + } + return [result[identity] for identity in expected] -def _metric(caught: int, total: int) -> str: - """Format a binary row-coverage count without inventing an empty rate.""" +def _rate(part: str, caught: int, total: int) -> str: + """Explain each denominator, including when there is nothing to score.""" + if part == "Overall": + return f"Overall detection rate: Of all {total} Human findings, either part of the tool caught {caught} ({caught / total:.1%})." + label = part.removesuffix(" checks") if total == 0: - return "N/A (0 eligible rows)" - return f"{caught}/{total} ({caught / total:.1%})" + return f"{label} detection rate: No Human findings were expected to be detected by the {part} from the page's HTML." + subject = "it" if part == "LLM" else "they" + return f"{label} detection rate: Of the {total} Human findings the {part} could be expected to detect from the page's HTML, {subject} caught {caught} ({caught / total:.1%})." def _md(value: object) -> str: """Escape inline text for Markdown, including raw HTML, keeping line breaks.""" - text = str(value).replace("\\", "\\\\").replace("|", "\\|") - text = text.replace("<", "\\<").replace(">", "\\>") + text = str(value) + for character in "\\`*_[]|<>": + text = text.replace(character, "\\" + character) return text.replace("\n", "
") -def _evidence_block(evidence: dict[str, str]) -> list[str]: - """Render source evidence verbatim in a collapsible fenced JSON block.""" - body = json.dumps(evidence, ensure_ascii=False, indent=2) - longest = max((len(run) for run in re.findall(r"`+", body)), default=0) - fence = "`" * max(3, longest + 1) - return [ - "
Original defect evidence", - "", - f"{fence}json", - body, - fence, - "", - "
", - ] - - def write_report( path: Path, manifest: dict[str, Any], rows: list[dict[str, Any]], llm: list[dict[str, Any]], programmatic: list[dict[str, Any]], + reviewer: str, + pricing_date: str, ) -> None: - """Write the single human-readable score and row evidence projection.""" + """Explain the review's catches, misses, and scoring in source order.""" if not manifest["complete"]: raise ValueError("Incomplete Evaluation runs cannot produce a final report") findings = {item["finding_id"]: item for item in llm + programmatic} - llm_rows = [row for row in rows if row["decision"]["classification"] == "llm_eligible"] - checks = [ - row for row in rows if row["decision"]["classification"] == "programmatic" - ] + expected = { + "llm_eligible": "LLM", + "programmatic": "Programmatic checks", + "unavailable_evidence": "Not testable", + "ambiguous": "Not testable", + } covered = [row for row in rows if any(row["matches"].values())] - llm_caught = sum(bool(row["matches"]["llm"]) for row in llm_rows) - programmatic_caught = sum(bool(row["matches"]["programmatic"]) for row in checks) - unavailable = sum( - row["decision"]["classification"] == "unavailable_evidence" for row in rows - ) - ambiguous = sum(row["decision"]["classification"] == "ambiguous" for row in rows) + usage = manifest["usage"] + tokens = [f"{usage['input_tokens']} input tokens", f"{usage['output_tokens']} output tokens"] + for key, label in (("cached_input_tokens", "cached input"), ("cache_creation_input_tokens", "cache-creation input")): + if usage[key]: + tokens.append(f"{usage[key]} {label} tokens") lines = [ - f"# Evaluation run {manifest['run_id']}", + f"# LLM accessibility evaluation: {_md(manifest['source_url'])}", "", - f"- Model: {manifest['configuration']['model']}", - f"- LLM detection rate: {_metric(llm_caught, len(llm_rows))}", - f"- Programmatic detection rate: {_metric(programmatic_caught, len(checks))}", - f"- Overall detection rate: {_metric(len(covered), len(rows))}", - f"- Estimated run cost: ${manifest['estimated_cost_usd']}", - f"- Unavailable evidence: {unavailable}; ambiguous: {ambiguous}", - f"- Reported usage: {_md(json.dumps(manifest['usage'], sort_keys=True))}", + "## How to read this report", "", - ( - "This is full-row coverage of this homepage Human audit only. " - "A blank finding cell means missed." - ), - ( - "The cost guardrail is checked between requests; " - "a request can exceed the remaining amount." - ), - ( - "Malformed response prompts: " - f"{_md(', '.join(manifest['format_failures']) or 'none')}" - ), + "The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.", + "The tool has two parts. The LLM asks a large language model to judge accessibility problems. Programmatic checks apply fixed rules without AI.", + "The tool saw only saved HTML, not a live browser or screen reader. This evaluation covers the homepage only, including Global findings that apply there.", + "A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.", "", - "## Human findings", + "## Summary", + "", + f"Model: {_md(manifest['configuration']['model'])}. Run ID: {_md(manifest['run_id'])}.", + f"Review decisions by: {_md(reviewer)}", "", ] + for classification, kind, part in ( + ("llm_eligible", "llm", "LLM"), + ("programmatic", "programmatic", "Programmatic checks"), + ): + eligible = [row for row in rows if row["decision"]["classification"].strip() == classification] + lines.extend([_rate(part, sum(bool(row["matches"][kind]) for row in eligible), len(eligible)), ""]) + lines.extend([ + _rate("Overall", len(covered), len(rows)), + "", + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from {', '.join(tokens)} multiplied by the published prices dated {_md(pricing_date)}. The actual bill may differ slightly.", + "", + ]) + if manifest["format_failures"]: + checks = ", ".join(_md(name.replace("_", " ") + " check") for name in manifest["format_failures"]) + lines.extend([f"The LLM returned unreadable output for {len(manifest['format_failures'])} of {len(manifest['prompts'])} checks: {checks}.", ""]) + lines.extend([ + "## Human findings", "", + "| Issue Title | WCAG | Expected to be caught by | LLM | Programmatic checks |", + "| --- | --- | --- | --- | --- |", + ]) for row in rows: - human_finding = row["human_finding"] - decision = row["decision"] - matches = row["matches"] - lines.extend( - [ - f"### {human_finding['human_finding_id']} — {_md(human_finding['problem'])}", - "", - ( - f"- Source: {human_finding['source_sheet']} " - f"row {human_finding['source_row']} ({human_finding['page_scope']})" - ), - f"- source_element: {_md(human_finding['source_element'])}", - ( - f"- Classification: {decision['classification']} — " - f"{_md(decision['classification_reason'])}" - ), - f"- LLM findings: {_md('; '.join(matches['llm']) or 'none')}", - ( - "- Programmatic findings: " - f"{_md('; '.join(matches['programmatic']) or 'none')}" - ), - f"- Match reason: {_md(decision['match_reason'] or 'none')}", - f"- Review notes: {_md(decision['review_notes'] or 'none')}", - ] - ) - for finding_id in (*matches["llm"], *matches["programmatic"]): - finding = findings[finding_id] - lines.append( - f"- {finding_id}: {_md(finding['problem'])} " - f"@ {_md(finding['location'])}" - ) - lines.extend(["", *_evidence_block(human_finding["raw_evidence"]), ""]) + human = row["human_finding"] + marks = ["✓" if row["matches"][kind] else "✗" for kind in ("llm", "programmatic")] + lines.append(f"| {_md(human['problem'])} | {_md(human['raw_evidence']['WCAG Sc'])} | {expected[row['decision']['classification'].strip()]} | {' | '.join(marks)} |") + lines.extend([ + "", + "The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part still gets a ✓ and counts toward the Overall detection rate, which counts each Human finding once out of all imported Human findings.", + ]) + groups = { + "What each part of the tool caught": covered, + "Missed": [ + row for row in rows + if expected[row["decision"]["classification"].strip()] != "Not testable" + and not any(row["matches"].values()) + ], + "Not testable from the page file": [ + row for row in rows + if expected[row["decision"]["classification"].strip()] == "Not testable" + ], + } + for heading, group in groups.items(): + lines.extend(["", f"## {heading}", ""]) + if heading == "Not testable from the page file": + lines.extend(["These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator.", ""]) + for row in group: + decision = row["decision"] + matches = row["matches"] + lines.extend([f"### {_md(row['human_finding']['problem'])}", ""]) + for kind in ("llm", "programmatic"): + for identity in matches[kind]: + finding = findings[identity] + label = ( + f"LLM ({finding['prompt'].replace('_', ' ')} check)" + if kind == "llm" else f"Programmatic check {finding['prompt']}" + ) + lines.extend([f"- {_md(label)}: {_md(finding['problem'])}", f" Why this counts: {_md(decision['match_reason'])}"]) + if heading == "Not testable from the page file": + lines.append(_md(decision["classification_reason"])) + elif not any(matches.values()): + for field, label in ( + ("classification_reason", "Why it was expected to be caught"), + ("review_notes", "Why it was missed"), + ): + if decision[field].strip(): + lines.append(f"{label}: {_md(decision[field])}") + lines.append("") path.write_text("\n".join(lines) + "\n", encoding="utf-8") From bffab02709d06dc67b456682f3c318982feba4af Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 17:44:11 -0400 Subject: [PATCH 049/126] chore(tickets): resolve plain-language report ticket --- .../issues/02-plain-language-report.md | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/.scratch/plain-language-report/issues/02-plain-language-report.md b/.scratch/plain-language-report/issues/02-plain-language-report.md index 68fb779..50c7fb9 100644 --- a/.scratch/plain-language-report/issues/02-plain-language-report.md +++ b/.scratch/plain-language-report/issues/02-plain-language-report.md @@ -14,25 +14,62 @@ It contains no IDs, CSS paths, location JSON, raw evidence JSON or raw usage JSO **Blocked by:** 01 (Rename to Human audit and LLM vocabulary) -**Status:** ready-for-agent +**Status:** resolved -- [ ] `report` fails without `--reviewer`; with it, the Summary shows "Review decisions by: " -- [ ] The title is "LLM accessibility evaluation: ", and the Summary shows the model and run ID -- [ ] "How to read this report" says: +- [x] `report` fails without `--reviewer`; with it, the Summary shows "Review decisions by: " +- [x] The title is "LLM accessibility evaluation: ", and the Summary shows the model and run ID +- [x] "How to read this report" says: - the benchmark is the Human audit; - the tool has two parts (the LLM and programmatic checks), each defined; - the tool saw only the saved HTML; - a reviewer set expectations and only full coverage counts; - the scope is the homepage only -- [ ] The LLM, Programmatic and Overall detection rates each appear as a sentence with the count, the denominator explained, and a percentage; zero eligible findings is stated in words -- [ ] The cost line shows dollars rounded to cents, input and output tokens, and the pricing-schedule publication date, and ends "The actual bill may differ slightly." Cached and cache-creation tokens appear only when non-zero -- [ ] A sentence names the LLM checks with malformed output out of the total prompt count, and is omitted when there are none -- [ ] The table has one row per Human finding in source order, with columns for the Issue Title, WCAG, expected part (LLM / Programmatic checks / Not testable), and ✓/✗ for each part; cross-catches are marked, and a footnote explains what the rates count -- [ ] Each caught Human finding lists its matched findings, labelled "LLM ( check)" or "Programmatic check ", with the problem text and "Why this counts:" giving the match reason -- [ ] "Missed" lists every `llm_eligible` or `programmatic` Human finding with no match, even when the classification reason and review notes are both empty; "Why it was expected to be caught" and "Why it was missed" appear only when non-empty -- [ ] "Not testable from the page file" groups `unavailable_evidence` and `ambiguous` findings under one intro sentence, each with its classification reason -- [ ] The report contains no Human finding or finding IDs, `source_element`, CSS paths or JSON blocks, and free text stays escaped for Markdown (inline HTML and table pipes) -- [ ] Tests through the `main` seam cover each item above, including a cross-catch, a missed finding with no explanations, both not-testable classifications, and the zero-eligible wording -- [ ] The README's description of `report` and its output matches the new layout, including the `--reviewer` argument +- [x] The LLM, Programmatic and Overall detection rates each appear as a sentence with the count, the denominator explained, and a percentage; zero eligible findings is stated in words +- [x] The cost line shows dollars rounded to cents, input and output tokens, and the pricing-schedule publication date, and ends "The actual bill may differ slightly." Cached and cache-creation tokens appear only when non-zero +- [x] A sentence names the LLM checks with malformed output out of the total prompt count, and is omitted when there are none +- [x] The table has one row per Human finding in source order, with columns for the Issue Title, WCAG, expected part (LLM / Programmatic checks / Not testable), and ✓/✗ for each part; cross-catches are marked, and a footnote explains what the rates count +- [x] Each caught Human finding lists its matched findings, labelled "LLM ( check)" or "Programmatic check ", with the problem text and "Why this counts:" giving the match reason +- [x] "Missed" lists every `llm_eligible` or `programmatic` Human finding with no match, even when the classification reason and review notes are both empty; "Why it was expected to be caught" and "Why it was missed" appear only when non-empty +- [x] "Not testable from the page file" groups `unavailable_evidence` and `ambiguous` findings under one intro sentence, each with its classification reason +- [x] The report contains no Human finding or finding IDs, `source_element`, CSS paths or JSON blocks, and free text stays escaped for Markdown (inline HTML and table pipes) +- [x] Tests through the `main` seam cover each item above, including a cross-catch, a missed finding with no explanations, both not-testable classifications, and the zero-eligible wording +- [x] The README's description of `report` and its output matches the new layout, including the `--reviewer` argument ## Comments + +- Operator correction: allow empty `classification_reason` values. A valid + classification is still required for every Human finding, and a match reason + is still required whenever a catch is claimed. The CLI test includes a missed + finding with both classification reason and review notes blank. The source + spec remains unchanged. +- Implemented in `be15c06e50b07e4033e278c4ac79cfd8248c51ba`. + `report` requires a nonblank reviewer description and emits the seven specified + sections with source-order table rows, cross-catches, matched problem text and + reasons, all misses, and the combined not-testable group. It omits detailed IDs, + source elements, locations, and raw JSON, while escaping free text. The README + documents the layout and optional classification reasons. +- Scoring clarification: the spec's explicit unchanged scoring rule takes + precedence over its contradictory exclusion prose. Not-testable findings stay + outside the LLM and Programmatic rates and remain in the Overall denominator; + the report and README say so. +- Verification: `uv run pytest -q` passed all 37 tests. The existing CLI suite + covers reviewer requirement and attribution; explanatory prose and section + order; 1/1 LLM, 1/1 Programmatic, and 2/3 Overall rates; both zero-eligible + sentences; shuffled CSV source ordering; matched source labels and reasons; + cross-catches; blank and explained misses; both not-testable classifications; + inline HTML and table-pipe escaping; omitted evidence and location fields; + nonzero cost rounded to cents, pricing date, cache counts present and absent; + and unreadable-output wording present and absent. Existing classification, + finding-ID, and required match-reason validation remain covered. + `uv run python -m compileall -q vision_aid/evaluation tests/test_evaluation_cli.py` + and `git diff --check` passed. No repository typechecker is configured; an + isolated targeted mypy check also passed for the two changed production files. +- Parallel code review against `61ac26c01fefdfc9abd5bf5ad19f981a728acfd9`: + Standards found no documented violations and one optional, low-severity + Primitive Obsession observation about two local section-title comparisons. + Kept the small renderer without added abstraction. Spec found zero findings; + the independent reviewer also reran the CLI suite, with 8 tests passing. +- Python additions against main: 1,585 production lines and 2,593 including + tests, versus the original 1,562 / 2,534 baseline. The report replaces old + verbosity and reuses existing CLI fixtures and tests. No paid calls made in + this ticket; live verification belongs to ticket 03. From e77b2c06efa9b17577605e933d5048cf6e159061 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 18:02:04 -0400 Subject: [PATCH 050/126] chore(evaluation): record reviewed live verification --- .../issues/03-live-verification-run.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.scratch/plain-language-report/issues/03-live-verification-run.md b/.scratch/plain-language-report/issues/03-live-verification-run.md index 484a79b..55f0a66 100644 --- a/.scratch/plain-language-report/issues/03-live-verification-run.md +++ b/.scratch/plain-language-report/issues/03-live-verification-run.md @@ -8,8 +8,21 @@ - [ ] One live run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with `--max-cost-usd 1.10` completes, or its stop reason is reported without a retry - [ ] A sub-agent fills in every `review.csv` row. It classifies each Human finding against the snapshot before reading the LLM responses, then records Match decisions following the README's rules, with a reason for every classification and match -- [ ] `report` succeeds with `--reviewer "Claude agent (operator-directed)"` +- [ ] `report` succeeds with `--reviewer "Codex agent (operator-directed)"` - [ ] The generated report is read end to end, and any wording that still needs knowledge of the evaluator is reported to the operator - [ ] The run's estimated cost and detection rates are recorded under Comments ## Comments + +2026-09-29 verification: + +- Operator corrections: attribute the available reviewer as Codex, and replace the saved homepage HTML before calculating its new checksum. The source spec and historical plan remain unchanged. +- Refreshed snapshot: 184,244 bytes, SHA-256 `a0b5f43b37ffa276b23d04da94e846c24ec2b2148ec9152ec448db7e68945c5a`. Human audit SHA-256 remains `f8e9c83d5de8155cd81c4d9c15180c6e38640a5aba7323d35040b81e39aa6c9a`. +- Report and reviewed CSV: `.model-evaluation/runs/20260929T215031-9f661935/{report.md,review.csv}`. All 16 rows reviewed by a fresh sub-agent; baseline frozen at 21:50:16 UTC before dispatch, SHA-256 `f7c1077a3e8441b1cf4e8b237145e0eb5cca550db6868a820d9b5a3d4c04ca19`. +- Independent classifications: 8 LLM-eligible, 4 programmatic, 4 unavailable, 0 ambiguous. Unlike the historical 6/6/4 plan, the complete heading-hierarchy defect and decorative text-link defect require contextual judgment. Classifications/reasons stayed frozen after dispatch; full matches require reasons and never reuse finding IDs. +- One live attempt used `--max-cost-usd 1.10`, fixed Haiku, 16,000 thinking tokens, 24,192 output cap, sequential requests and zero retries. Free counting found 7,421 input tokens; exact prepared prompts matched before paid dispatch. The private ledger reserved $1.50 for this attempt and accounted for $0.237422 of prior spending. +- All 8 requests succeeded with `end_turn`; none reached the output cap. SVG output contained valid fenced JSON followed by prose and was recorded as a format failure, without repair or retry. All raw replies and 16 LLM/29 Programmatic findings were inspected. +- Estimated cost: $0.104626 for 7,421 input and 19,441 output tokens, no cache usage, using the recorded 2026-09-22 prices reverified on 2026-09-29. Reconciled cumulative estimate $0.342048; remaining budget $4.657952. +- LLM detection 1/8 (12.5%); Programmatic detection 4/4 (100%); Overall detection 5/16 (31.25%, displayed 31.2%). Matches: Human audit findings 1, 2, 3, 7, 12; eligible misses 4, 5, 6, 8, 9, 10, 16; not testable 11, 13, 14, 15. +- `report --reviewer "Codex agent (operator-directed)"` succeeded. Reviewer, implementation agent and operator agent read it end to end; CSS selectors and internal review-phase wording were removed from live notes. Remaining reader limitations: "unreadable" means a parsing failure here; bundled findings include unrelated issues, repeat match reasons and make navigation credit unclear without knowing the no-reuse rule. Some frozen explanations retain accessibility/HTML terminology. +- Verification: `uv run pytest -q` (37 passed), `uv lock --check`, entry-point imports, and independent CSV/source/identity/rate/report checks passed. No production code or tests added. Snapshot, baseline, spending ledger and full run artifacts remain private and ignored. From 2fb259af58dc7d00b88866d65297cd10797ced8d Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Tue, 29 Sep 2026 18:04:28 -0400 Subject: [PATCH 051/126] chore(tracker): resolve live verification ticket --- .../issues/03-live-verification-run.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.scratch/plain-language-report/issues/03-live-verification-run.md b/.scratch/plain-language-report/issues/03-live-verification-run.md index 55f0a66..8dd7e60 100644 --- a/.scratch/plain-language-report/issues/03-live-verification-run.md +++ b/.scratch/plain-language-report/issues/03-live-verification-run.md @@ -4,13 +4,13 @@ **Blocked by:** 02 (Plain-language report) -**Status:** ready-for-agent +**Status:** resolved -- [ ] One live run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with `--max-cost-usd 1.10` completes, or its stop reason is reported without a retry -- [ ] A sub-agent fills in every `review.csv` row. It classifies each Human finding against the snapshot before reading the LLM responses, then records Match decisions following the README's rules, with a reason for every classification and match -- [ ] `report` succeeds with `--reviewer "Codex agent (operator-directed)"` -- [ ] The generated report is read end to end, and any wording that still needs knowledge of the evaluator is reported to the operator -- [ ] The run's estimated cost and detection rates are recorded under Comments +- [x] One live run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with `--max-cost-usd 1.10` completes, or its stop reason is reported without a retry +- [x] A sub-agent fills in every `review.csv` row. It classifies each Human finding against the snapshot before reading the LLM responses, then records Match decisions following the README's rules, with a reason for every classification and match +- [x] `report` succeeds with `--reviewer "Codex agent (operator-directed)"` +- [x] The generated report is read end to end, and any wording that still needs knowledge of the evaluator is reported to the operator +- [x] The run's estimated cost and detection rates are recorded under Comments ## Comments @@ -26,3 +26,5 @@ - LLM detection 1/8 (12.5%); Programmatic detection 4/4 (100%); Overall detection 5/16 (31.25%, displayed 31.2%). Matches: Human audit findings 1, 2, 3, 7, 12; eligible misses 4, 5, 6, 8, 9, 10, 16; not testable 11, 13, 14, 15. - `report --reviewer "Codex agent (operator-directed)"` succeeded. Reviewer, implementation agent and operator agent read it end to end; CSS selectors and internal review-phase wording were removed from live notes. Remaining reader limitations: "unreadable" means a parsing failure here; bundled findings include unrelated issues, repeat match reasons and make navigation credit unclear without knowing the no-reuse rule. Some frozen explanations retain accessibility/HTML terminology. - Verification: `uv run pytest -q` (37 passed), `uv lock --check`, entry-point imports, and independent CSV/source/identity/rate/report checks passed. No production code or tests added. Snapshot, baseline, spending ledger and full run artifacts remain private and ignored. + +Resolved after implementation commit `e77b2c06efa9b17577605e933d5048cf6e159061`. Independent Standards and Spec reviews passed with zero findings; all acceptance criteria demonstrated by the verification recorded above. This separate closure commit records the exact implementation ID. From b8e3a36642d973eda26b3f631783b27cc6c336fc Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 09:52:00 -0400 Subject: [PATCH 052/126] fix(evaluation): clarify report checks and format explanation bullets --- tests/test_evaluation_cli.py | 12 ++++++------ vision_aid/evaluation/review.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 42dd764..01e27e5 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -230,8 +230,8 @@ def client(**options: object) -> FakeClient: headings = ["How to read this report", "Summary", "Human findings", "What each part of the tool caught", "Missed", "Not testable from the page file"] assert [report.index("## " + heading + "\n") for heading in headings] == sorted(report.index("## " + heading + "\n") for heading in headings) assert all(text in report for text in ( - "benchmark is the Human audit", "The tool has two parts", "large language model", - "fixed rules without AI", "only saved HTML", "homepage only", "A reviewer decided", + "benchmark is the Human audit", "This API uses LLM checks and Programmatic checks", + "send extracted page content to the model", "apply fixed rules directly", "only saved HTML", "homepage only", "A reviewer decided", "fully cover the whole Human finding", "Partial coverage earns no credit", )) assert "Review decisions by: Codex agent (operator-directed)" in report @@ -250,7 +250,7 @@ def client(**options: object) -> FakeClient: assert "| Missing destination | 2.4.4 | Programmatic checks | ✗ | ✓ |" in table assert "LLM (link clarity check): Unclear link" in report assert "Programmatic check " + programmatic[0]["prompt"].replace("_", r"\_") in report - assert "Why this counts: Full issue" in report + assert "\n\n - Why this counts: Full issue" in report assert "Visual context" in report assert all(row["human_finding_id"] not in report for row in rows) assert all(item["finding_id"] not in report for item in findings + programmatic) @@ -309,7 +309,7 @@ def call(self, prompt: str) -> dict: report = (run / "report.md").read_text() assert r"Unclear \\|link" in report assert r"Use \\|landmark" in report - assert r"Why this counts: Full \\|coverage" in report + assert "\n\n - " + r"Why this counts: Full \\|coverage" in report assert "| Missing destination | 2.4.4 | Programmatic checks | ✓ | ✗ |" in report assert "it caught 1 (100.0%)" in report and "they caught 0 (0.0%)" in report assert "Of all 5 Human findings, either part of the tool caught 2 (40.0%)" in report @@ -331,8 +331,8 @@ def call(self, prompt: str) -> dict: write_review(review, rows) assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 explained = (run / "report.md").read_text().split("## Missed\n", 1)[1] - assert "Why it was expected to be caught: The image is in the HTML" in explained - assert "Why it was missed: No full finding" in explained + assert "\n- Why it was expected to be caught: The image is in the HTML" in explained + assert "\n- Why it was missed: No full finding" in explained def test_review_requires_classification_and_rejects_invalid_finding_ids( diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 3d7a55f..08f2e50 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -169,7 +169,7 @@ def write_report( "## How to read this report", "", "The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.", - "The tool has two parts. The LLM asks a large language model to judge accessibility problems. Programmatic checks apply fixed rules without AI.", + "This API uses LLM checks and Programmatic checks. LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.", "The tool saw only saved HTML, not a live browser or screen reader. This evaluation covers the homepage only, including Global findings that apply there.", "A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.", "", @@ -234,7 +234,7 @@ def write_report( f"LLM ({finding['prompt'].replace('_', ' ')} check)" if kind == "llm" else f"Programmatic check {finding['prompt']}" ) - lines.extend([f"- {_md(label)}: {_md(finding['problem'])}", f" Why this counts: {_md(decision['match_reason'])}"]) + lines.extend([f"- {_md(label)}: {_md(finding['problem'])}", "", f" - Why this counts: {_md(decision['match_reason'])}"]) if heading == "Not testable from the page file": lines.append(_md(decision["classification_reason"])) elif not any(matches.values()): @@ -243,6 +243,6 @@ def write_report( ("review_notes", "Why it was missed"), ): if decision[field].strip(): - lines.append(f"{label}: {_md(decision[field])}") + lines.append(f"- {label}: {_md(decision[field])}") lines.append("") path.write_text("\n".join(lines) + "\n", encoding="utf-8") From 1bc12d995498e54a61b1a0ab338402cd95e74084 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 10:22:09 -0400 Subject: [PATCH 053/126] feat(evaluation)!: organize supporting run artifacts Keep review evidence, run status, and scored results at the run root. Store canonical Human findings, pricing, raw outputs, prompts, and payloads under artifacts/. Preserve scoring and per-request persistence. BREAKING CHANGE: reporting reads only the new layout. Existing run directories require manual reorganization. --- .scratch/run-directory-layout/spec.md | 137 +++++++++++++++++++++++ README.md | 63 ++++++----- tests/test_evaluation_cli.py | 152 ++++++++++++++++++++++++-- vision_aid/evaluation/cli.py | 21 ++-- vision_aid/evaluation/llm_audit.py | 7 +- 5 files changed, 333 insertions(+), 47 deletions(-) create mode 100644 .scratch/run-directory-layout/spec.md diff --git a/.scratch/run-directory-layout/spec.md b/.scratch/run-directory-layout/spec.md new file mode 100644 index 0000000..0bbe805 --- /dev/null +++ b/.scratch/run-directory-layout/spec.md @@ -0,0 +1,137 @@ +# Organize the Evaluation run directory for review and results + +Status: ready-for-agent + +## Problem Statement + +A human completing `review.csv` must sift through an Evaluation run directory that mixes review evidence with raw outputs, extracted payloads, prompts, pricing, and canonical Human finding data. Most of those supporting artifacts are unnecessary for ordinary Eligibility classifications and Match decisions. The separate Human finding JSON duplicates evidence already included in the CSV, while the files the reviewer needs are scattered among internal artifacts. + +The same root directory should remain useful after review, when the human reads the scored report. Clearing the root must also preserve access to run completion and malformed-response details, which currently appear in the run manifest and can affect how the reviewer interprets the findings. + +## Solution + +Keep the Evaluation run root focused on review and scored results. Leave the editable CSV, full Benchmark snapshot, both normalized finding lists, and the run manifest at the root. Generate the scored Markdown report there after successful review and reporting. Move supporting artifacts together into one `artifacts/` subdirectory, preserving their filenames. + +The directory contract is: + +| Artifact | Location | Purpose | +| --- | --- | --- | +| `review.csv` | Run root | Original Human finding evidence and editable human decisions | +| `snapshot.html` | Run root | Full Benchmark snapshot used to judge testability and coverage | +| `normalized-llm-findings.json` | Run root | LLM findings with copyable finding IDs | +| `normalized-programmatic-findings.json` | Run root | Programmatic findings with copyable finding IDs | +| `run.json` | Run root | Completion, malformed-response details, configuration, and provenance | +| `report.md` | Run root, after successful reporting | Scored results for the human reader | +| `human-findings.json` | `artifacts/` | Canonical imported Human findings for validation and scoring | +| `pricing.json` | `artifacts/` | Recorded price schedule for Estimated run cost | +| `raw-llm-responses.json` | `artifacts/` | Original LLM replies, usage, and request failures | +| `raw-programmatic-findings.json` | `artifacts/` | Original Programmatic check output | +| `prompts/` | Inside `artifacts/` | Saved prompts used by LLM execution | +| `payloads/` | Inside `artifacts/` | Extracted content used to prepare prompts | + +A complete live run has exactly five files and the supporting directory at its root before reporting. Successful reporting adds the Markdown report. Preview and interrupted runs keep their existing artifact availability and completion semantics, with the same location rules for the artifacts they produce. + +This is a clean break. New runs use only the new layout, and the updated report command reads only that layout. Existing run directories remain untouched and require manual reorganization before the updated report command can read them. + +## User Stories + +1. As a reviewer, I want the Evaluation run root to contain the files needed for review, so that I can identify my working evidence without sorting through internal artifacts. +2. As a reviewer, I want the review CSV to remain at the root, so that I can immediately find the file I must edit. +3. As a reviewer, I want the original Human finding evidence to remain in the CSV, so that I can review each finding without opening a duplicate JSON file. +4. As a reviewer, I want the Human audit's source sheet and row to remain in the CSV, so that I can trace the source evidence. +5. As a reviewer, I want the full Benchmark snapshot at the root, so that I can judge what the saved page can show. +6. As a reviewer, I want the normalized LLM findings at the root, so that I can inspect their evidence when making Match decisions. +7. As a reviewer, I want the normalized Programmatic findings at the root, so that I can inspect the fixed-rule evidence when making Match decisions. +8. As a reviewer, I want finding IDs to remain available in both normalized lists, so that I can copy valid IDs into the review CSV. +9. As a reviewer, I want existing artifact filenames preserved, so that the reorganization does not also require learning new names. +10. As a reviewer, I want the run manifest at the root, so that I can inspect run status before interpreting the findings. +11. As a reviewer, I want incomplete-run reasons preserved, so that I can understand why a run stopped. +12. As a reviewer, I want malformed-response details preserved, so that an empty finding list is not mistaken for evidence that a check found no problems. +13. As a reviewer, I want the scored report at the root after successful reporting, so that I can read the results where I performed the review. +14. As a report reader, I want the report's content unchanged, so that reorganizing files does not change the explanation of the results. +15. As a reviewer, I want supporting artifacts together in one subdirectory, so that they remain discoverable without crowding the root. +16. As a reviewer investigating a result, I want access to the original LLM replies, so that I can inspect missing or malformed output. +17. As a reviewer investigating a result, I want access to raw Programmatic findings, so that I can inspect the original check output. +18. As a reviewer investigating a result, I want access to saved prompts, so that I can see the evidence and instructions sent to the LLM. +19. As a reviewer investigating a result, I want access to extracted payloads, so that I can investigate preprocessing differences from the full Benchmark snapshot. +20. As an evaluator, I want canonical Human finding data retained under supporting artifacts, so that report validation remains reproducible without duplicating everyday reviewer context at the root. +21. As an evaluator, I want the saved price schedule retained under supporting artifacts, so that Estimated run cost remains traceable to its recorded prices. +22. As an operator, I want preview runs to use the new layout, so that file placement is consistent before and after live execution. +23. As an operator, I want preview runs to retain their existing restriction on paid requests, so that reorganizing files does not change Live-run approval. +24. As an operator, I want declined live execution to retain its existing preview evidence and status, so that I can inspect the prepared run. +25. As an operator, I want stopped runs to retain responses and usage in the new locations, so that request failures and exhausted cost guardrails remain inspectable. +26. As an operator, I want evidence saved after each attempted prompt as before, so that an interruption does not lose evidence already gathered. +27. As an evaluator, I want the run's identity to remain the root directory's name, so that the supporting directory is never mistaken for an Evaluation run. +28. As an evaluator, I want normalized findings to retain the correct run identity and source URL, so that evidence validation continues to reject unrelated findings. +29. As an evaluator, I want reporting to read supporting data from its new location, so that I can score reviews of newly generated runs. +30. As an evaluator, I want Eligibility classification and Match decision validation unchanged, so that moving evidence does not alter what counts as coverage. +31. As an evaluator, I want all detection rates unchanged for the same reviewed evidence, so that results remain comparable. +32. As an operator, I want existing run directories left untouched, so that this change does not move, overwrite, or delete prior evidence. +33. As a maintainer, I want the report command to read only the new layout, so that there is one supported artifact contract. +34. As an operator, I want the clean break documented, so that I know an old run needs manual reorganization before reporting. +35. As a maintainer, I want the shared pipeline's output layout outside evaluation preserved, so that the reviewer-focused change does not affect other pipeline users. +36. As a reviewer, I want review and troubleshooting instructions to use the new locations, so that the documented workflow matches generated runs. +37. As a maintainer, I want lifecycle tests to verify the root contents and supporting directory, so that internal artifacts cannot silently reappear at the root. +38. As a maintainer, I want verification through the existing CLI with a fake provider, so that the layout and scoring can be checked without paid execution or new testing interfaces. + +## Implementation Decisions + +- Modify the evaluation CLI's preparation and reporting behavior and the LLM execution module's artifact reads and writes. Retain the existing command arguments and run-directory interface. +- Use the directory contract in the Solution as the externally visible artifact layout. Preserve existing filenames, artifact schemas, CSV columns, finding ID formats, and report content. +- Create the supporting directory before writing or copying artifacts into it. Keep prompts and payloads nested inside that directory. +- Direct the evaluation use of the shared pipeline to the supporting directory for its outputs. Keep the full Benchmark snapshot and normalized Programmatic findings at the root, and preserve the existing cleanup of temporary pipeline output. +- Keep the Evaluation run root distinct from the pipeline output directory. Run manifests and normalized findings must retain the root directory's identity, not the name of the supporting directory. +- LLM execution reads the moved pricing and saved prompts, writes raw responses under supporting artifacts, and continues writing normalized LLM findings and the run manifest at the root. +- Preserve existing per-prompt persistence, response normalization, provider configuration, usage accounting, cost guardrails, and completion behavior. Malformed successful replies retain their current recorded status and normalized-finding behavior. +- Reporting reads canonical Human findings and pricing from supporting artifacts while reading the review CSV, normalized findings, and manifest from the root. Successful reporting writes the report at the root. +- Preserve Human audit identity checks, run identity checks, finding-source checks, source URL checks, review validation, and deterministic scoring. +- Apply the location rules to preview, declined execution, complete execution, and incomplete execution without creating artifacts earlier than their existing lifecycle stage. +- New generation and reporting support only the new layout. Add no old-layout fallback, migration command, or automatic changes to existing runs. +- Update the README's evaluation workflow, artifact table, review instructions, and troubleshooting locations. Explain that existing runs require manual reorganization before the updated report command can read them. +- Preserve the shared pipeline's usual output layout for callers outside evaluation. + +## Testing Decisions + +- Use one existing highest-level seam: the evaluation CLI's `main` entry point, with the existing LLM request client replaced by a fake for live-execution cases. The operator confirmed this lifecycle-testing approach with the complete plan. +- Good tests exercise generated files, retained evidence, CLI outcomes, and report results. They do not assert on internal path helpers, directory-creation call order, or implementation-specific module structure. No new test seam is needed. +- Prior art is the existing evaluation CLI lifecycle suite: it builds a small Human audit and Benchmark snapshot, runs preview or fake live execution, edits the generated CSV, and invokes reporting. It already checks hand-calculated detection rates, review validation, source evidence, run location, and request-failure and cost-guardrail behavior. +- Extend the preview test to check the new locations, preserved source evidence, retained root manifest and normalized Programmatic findings, absent live-only artifacts, no provider calls, and continued refusal to report an incomplete run. +- Extend the successful fake-live lifecycle test to assert the exact pre-report root entries, all supporting entries in their selected locations, and absence of duplicate supporting files at the root. Edit the root CSV using IDs from the root normalized findings, report successfully, and assert that only the report is added to the root. +- Preserve existing report and review-validation assertions, including reviewer attribution and hand-calculated LLM, Programmatic, and Overall detection rates. These verify that reporting can read the moved canonical evidence and price schedule without changing scoring. +- Verify that manifest and finding run IDs still match the run root's name and that existing checks reject evidence belonging to another run. Preserve the existing test for invocation from a worktree subdirectory. +- Extend stopped-run cases to verify raw responses and usage remain available in the selected locations, the root manifest retains the incomplete reason, normalized evidence is retained, and final reporting remains refused. +- Verify malformed successful replies retain their recorded format-failure details at the root and raw responses under supporting artifacts without changing completion semantics or inventing normalized matches. +- Verify per-prompt persistence through the fake provider boundary: before a subsequent request, evidence from the preceding request is already readable at the selected locations. Retain final saved evidence when execution stops. +- Verify reporting has no fallback to old root-level supporting files, using disposable test evidence rather than changing private run directories. Existing directories must not be automatically reorganized. +- Run the evaluation CLI lifecycle suite and the repository's applicable existing checks. Reuse existing shared-pipeline checks to confirm its ordinary output contract remains unchanged. +- No paid live verification is part of this spec. + +## Out of Scope + +- Generated reviewer guides, readable finding references, or replacement review interfaces. +- New status summaries, CLI warning behavior, filename changes, or artifact schema changes. +- Changes to Eligibility classifications, Match decisions, review validation, detection rates, model ranking, or report content. +- Changes to preprocessing, prompt content, provider configuration, pricing rules, cost guardrails, or Live-run approval. +- Migration commands, compatibility fallbacks, deletion, or automatic reorganization of existing runs. +- Relocating extra files created by historical one-off experiments. +- Changes to the shared pipeline's output layout outside evaluation. +- Paid live execution and implementation of the feature during spec publication. + +## Further Notes + +The CSV already contains the original Human audit evidence; its canonical JSON remains necessary for scorer validation but need not occupy the review-focused root. Normalized findings retain source evidence, while raw replies remain useful for diagnosing malformed LLM output. + +The run manifest deliberately stays visible even though it also contains internal metadata. Moving it would require another way to expose completion and malformed-response status, and the agreed scope is reorganizing existing files. + +Reviewers must continue judging Eligibility classifications against the full Benchmark snapshot. Content omitted from extracted payloads or prompts is not made unavailable merely by that omission. The reorganization does not change this rule. + +The plan preserves the existing decisions to separate human review from deterministic scoring and to rank models by LLM detection rate with Estimated run cost as the tie-breaker. Artifact placement is cheap to reverse and does not warrant an additional ADR or glossary term. + +## Comments + +- Q1: The operator chose to support both review and scored results at the root. +- Q2: The operator chose to reorganize existing files only. +- Q3: The operator chose to keep the run manifest at the root. +- Q4: The operator chose one supporting artifact directory with existing filenames. +- Q5: The operator chose the new layout only, leaving existing directories untouched. +- Q6: The operator confirmed the complete plan and requested publication through the to-spec skill. diff --git a/README.md b/README.md index 15684fa..eecbadd 100644 --- a/README.md +++ b/README.md @@ -290,8 +290,8 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report failure or cost limit and start a new live run. Failed requests are not retried. Use `$RUN_DIR/normalized-llm-findings.json` and `$RUN_DIR/normalized-programmatic-findings.json` for finding IDs and evidence. - `$RUN_DIR/raw-llm-responses.json` has model responses; - `$RUN_DIR/raw-programmatic-findings.json` has the original checker output. + `$RUN_DIR/artifacts/raw-llm-responses.json` has model responses; + `$RUN_DIR/artifacts/raw-programmatic-findings.json` has the original checker output. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in Excel or a CSV-aware text editor. Review every Human finding and fill in @@ -312,38 +312,51 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report every missed finding with any recorded explanations, and findings not testable from the page file with their reasons. IDs, locations, and raw JSON stay in the run artifacts. Not-testable findings are left out of the LLM and - Programmatic rates but remain in the Overall denominator. Existing Evaluation - run directories are unsupported. Reporting rejects incomplete runs, missing - classifications, and invalid or reused finding IDs. + Programmatic rates but remain in the Overall denominator. Reporting rejects + incomplete runs, missing classifications, and invalid or reused finding IDs. ### Evaluation run files -The `run` command saves these files under `$RUN_DIR`. Without `--live`, it -prepares the prompts but makes no model requests. +The run root holds the review evidence and scored results. Supporting files +are together under `$RUN_DIR/artifacts/`. Without `--live`, the command prepares +the prompts but makes no model requests; neither LLM output file exists yet. +Successful reporting adds `report.md` at the root. + +New runs and reporting use only this layout. Existing run directories remain +untouched and require manual reorganization before the updated `report` command +can read them. Move their supporting files into `artifacts/`, including the +`prompts/` and `payloads/` directories, preserving filenames. There is no +compatibility fallback or automatic migration. | File | Role | | --- | --- | -| `human-findings.json` | The imported Home and Global Human findings, their original Human audit evidence, and stable IDs. | -| `payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | -| `payloads/cl02_payload.json` | Extracted forms, fields, and labels. | -| `payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | -| `prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | -| `raw-programmatic-findings.json` | Original output from the Programmatic checks. | -| `normalized-programmatic-findings.json` | Those checker results converted to findings with `finding_id` values for `review.csv`. | -| `raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | -| `normalized-llm-findings.json` | Parsed LLM findings with `finding_id` values for `review.csv`. A malformed reply can appear in `raw-llm-responses.json` without producing a finding here. | -| `run.json` | The run's configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | -| `pricing.json` | A copy of the versioned model rates used to estimate this run's cost from reported usage. | +| `review.csv` | Original Human audit evidence, source sheet and row, stable IDs, and the decision columns the reviewer edits. | +| `snapshot.html` | The full Benchmark snapshot used to judge testability and coverage. | +| `normalized-llm-findings.json` | Parsed LLM findings with `finding_id` values for `review.csv`. A malformed reply can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | +| `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | +| `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | +| `report.md` | Scored results generated after successful reporting. | +| `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | +| `artifacts/pricing.json` | The versioned model rates used to estimate this run's cost from reported usage. | +| `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | +| `artifacts/raw-programmatic-findings.json` | Original output from the Programmatic checks. | +| `artifacts/prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | +| `artifacts/payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | +| `artifacts/payloads/cl02_payload.json` | Extracted forms, fields, and labels. | +| `artifacts/payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | The payload files show extracted page data before filtering. The prompt files show what the model actually received after filtering and slicing. To review a -Human finding, compare `human-findings.json` with the two normalized findings -files and inspect `snapshot.html` for the full Benchmark evidence. Check -`raw-llm-responses.json` when an LLM finding is missing or malformed, and -check the prompt file to see whether the model received the relevant evidence. +Human finding, compare its original evidence in `review.csv` with the two +normalized findings files and inspect `snapshot.html` for the full Benchmark +evidence. Check +`artifacts/raw-llm-responses.json` when an LLM finding is missing or malformed, and +check the file under `artifacts/prompts/` to see whether the model received the +relevant evidence. Record decisions in `review.csv`; the `report` command reads that CSV and the -run evidence to write `report.md`. The pipeline briefly writes `manifest.json`, -but the Evaluation command removes it and keeps the relevant metadata in +run evidence to write `report.md`. The pipeline briefly writes +`artifacts/manifest.json`, but the Evaluation command removes it and keeps the +relevant metadata in `run.json`. Earlier private experiments may also contain `preflight.json` with advance @@ -406,7 +419,7 @@ uv run visionaid-evaluate run --html .model-evaluation/pristine-home.html \ ``` This needs no API key or approval. It creates an incomplete run with -`raw-programmatic-findings.json`, `normalized-programmatic-findings.json`, +`artifacts/raw-programmatic-findings.json`, `normalized-programmatic-findings.json`, LLM prompts, and `review.csv`. Use the normalized file's `finding_id` values for programmatic matches. It makes no Haiku requests and cannot produce a final report. `--max-cost-usd` must be positive but is not spent without diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 01e27e5..671b34d 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -101,19 +101,34 @@ def test_preview_preserves_human_findings_and_never_calls_provider( assert main(args(human_audit, html)) == 0 output = capsys.readouterr().out run = directory(tmp_path) + assert {path.name for path in run.iterdir()} == { + "review.csv", "snapshot.html", "normalized-programmatic-findings.json", + "run.json", "artifacts", + } + artifacts = run / "artifacts" + assert {path.name for path in artifacts.iterdir()} == { + "human-findings.json", "pricing.json", "raw-programmatic-findings.json", + "prompts", "payloads", + } + assert list((artifacts / "prompts").glob("*.json")) + assert {path.name for path in (artifacts / "payloads").iterdir()} == { + "cl01_payload.json", "cl02_payload.json", "cl03_payload.json", + } assert "claude-haiku-4-5-20251001" in output assert "16000" in output and "24192" in output and "$0.01" in output assert (run / "snapshot.html").read_bytes() == html.read_bytes() - raw = json.loads((run / "raw-programmatic-findings.json").read_text()) + raw = json.loads((run / "artifacts" / "raw-programmatic-findings.json").read_text()) normalized = json.loads((run / "normalized-programmatic-findings.json").read_text()) assert raw and [finding["raw_source"] for finding in normalized] == raw assert not (run / "programmatic_findings.json").exists() assert not (run / "programmatic-findings.json").exists() manifest = json.loads((run / "run.json").read_text()) + assert manifest["run_id"] == run.name + assert all(item["run_id"] == run.name for item in normalized) assert manifest["snapshot_sha256"] == hashlib.sha256(html.read_bytes()).hexdigest() assert manifest["human_audit_filename"] == human_audit.name assert manifest["human_audit_sha256"] == hashlib.sha256(human_audit.read_bytes()).hexdigest() - imported = json.loads((run / "human-findings.json").read_text()) + imported = json.loads((run / "artifacts" / "human-findings.json").read_text()) assert imported["human_audit_sha256"] == manifest["human_audit_sha256"] assert all(item["human_finding_id"].startswith("human-") for item in imported["human_findings"]) rows = read_review(run / "review.csv") @@ -160,6 +175,15 @@ class FakeClient: def call(self, prompt: str) -> dict: """Return a successful fake provider response with usage.""" + if self.calls: + saved = directory(tmp_path) + responses = json.loads((saved / "artifacts" / "raw-llm-responses.json").read_text()) + findings = json.loads((saved / "normalized-llm-findings.json").read_text()) + manifest = json.loads((saved / "run.json").read_text()) + assert len(responses) == self.calls + assert len(findings) == self.calls + assert manifest["usage"]["input_tokens"] == self.calls * 1000 + assert manifest["complete"] is False self.calls += 1 return { "success": True, @@ -188,10 +212,20 @@ def client(**options: object) -> FakeClient: assert client_options[0]["max_retries"] == 0 run = directory(tmp_path) assert fake.calls == 3 + root_entries = { + "review.csv", "snapshot.html", "normalized-llm-findings.json", + "normalized-programmatic-findings.json", "run.json", "artifacts", + } + assert {path.name for path in run.iterdir()} == root_entries + assert {path.name for path in (run / "artifacts").iterdir()} == { + "human-findings.json", "pricing.json", "raw-llm-responses.json", + "raw-programmatic-findings.json", "prompts", "payloads", + } + assert json.loads((run / "run.json").read_text())["run_id"] == run.name assert ( json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.006525" ) - assert (run / "raw-llm-responses.json").exists() + assert (run / "artifacts" / "raw-llm-responses.json").exists() assert all(not (run / name).exists() for name in ( "references.json", "raw-audit-responses.json", "normalized-audit-findings.json" )) @@ -199,6 +233,7 @@ def client(**options: object) -> FakeClient: programmatic_file = run / "normalized-programmatic-findings.json" programmatic = json.loads(programmatic_file.read_text()) assert findings and programmatic + assert all(item["run_id"] == run.name for item in findings + programmatic) assert all(item["source"] == "llm" and item["finding_id"].startswith("llm-") for item in findings) assert all(item["finding_id"].startswith("programmatic-") for item in programmatic) review = run / "review.csv" @@ -225,6 +260,7 @@ def client(**options: object) -> FakeClient: main(["report", "--run-dir", str(run), "--reviewer", " "]) write_review(review, list(reversed(rows))) assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 + assert {path.name for path in run.iterdir()} == root_entries | {"report.md"} report = (run / "report.md").read_text() assert "# LLM accessibility evaluation: https://example.test/" in report headings = ["How to read this report", "Summary", "Human findings", "What each part of the tool caught", "Missed", "Not testable from the page file"] @@ -260,9 +296,9 @@ def client(**options: object) -> FakeClient: write_review(review, rows) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - imported = json.loads((run / "human-findings.json").read_text()) + imported = json.loads((run / "artifacts" / "human-findings.json").read_text()) imported["human_audit_sha256"] = "changed" - (run / "human-findings.json").write_text(json.dumps(imported)) + (run / "artifacts" / "human-findings.json").write_text(json.dumps(imported)) with pytest.raises(SystemExit): main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) @@ -297,6 +333,13 @@ def call(self, prompt: str) -> dict: assert main(args(human_audit, html, "--live", "--approve-live")) == 0 run = directory(tmp_path) findings = json.loads((run / "normalized-llm-findings.json").read_text()) + manifest = json.loads((run / "run.json").read_text()) + responses = json.loads((run / "artifacts" / "raw-llm-responses.json").read_text()) + assert manifest["complete"] is True + assert manifest["format_failures"] == ["page_title"] + assert responses[0]["result"]["response"] == "not JSON" + assert len(findings) == 2 + assert all(item["prompt"] != "page_title" for item in findings) review = run / "review.csv" rows = read_review(review) rows[0].update(classification="llm_eligible", llm_finding_id=findings[0]["finding_id"], match_reason="Full
|coverage") @@ -438,9 +481,16 @@ def isatty(self) -> bool: monkeypatch.setattr("sys.stdin", DeclinedInput()) assert main(args(human_audit, html, "--live")) == 0 - assert ( - json.loads((directory(tmp_path) / "run.json").read_text())["complete"] is False - ) + run = directory(tmp_path) + manifest = json.loads((run / "run.json").read_text()) + assert manifest["complete"] is False + assert manifest["incomplete_reason"] == "preview" + assert {path.name for path in run.iterdir()} == { + "review.csv", "snapshot.html", "normalized-programmatic-findings.json", + "run.json", "artifacts", + } + assert (run / "artifacts" / "prompts").is_dir() + assert not (run / "artifacts" / "raw-llm-responses.json").exists() @pytest.mark.parametrize("failure", [True, False]) @@ -460,7 +510,7 @@ def call(self, prompt: str) -> dict: self.calls += 1 return { "success": not failure, - "response": "[]" if not failure else None, + "response": json.dumps([{"problem": "Unclear page title"}]) if not failure else None, "error": "synthetic failure" if failure else None, "usage": {"input_tokens": 100000, "output_tokens": 20}, } @@ -472,8 +522,90 @@ def call(self, prompt: str) -> dict: manifest = json.loads((run / "run.json").read_text()) assert fake.calls == 1 assert manifest["complete"] is False + assert manifest["incomplete_reason"] == ( + "synthetic failure" if failure else "cost guardrail exhausted" + ) + assert {path.name for path in run.iterdir()} == { + "review.csv", "snapshot.html", "normalized-llm-findings.json", + "normalized-programmatic-findings.json", "run.json", "artifacts", + } + findings = json.loads((run / "normalized-llm-findings.json").read_text()) + assert len(findings) == (0 if failure else 1) + assert all(item["run_id"] == run.name for item in findings) assert manifest["usage"]["input_tokens"] == 100000 assert float(manifest["estimated_cost_usd"]) > 0 - assert len(json.loads((run / "raw-llm-responses.json").read_text())) == 1 + assert len(json.loads((run / "artifacts" / "raw-llm-responses.json").read_text())) == 1 with pytest.raises(SystemExit): main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) + + +@pytest.mark.parametrize("filename", ["human-findings.json", "pricing.json"]) +def test_report_requires_supporting_artifacts_without_reorganizing_old_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], filename: str, +) -> None: + """Root-level evidence is neither a fallback nor an automatic migration.""" + human_audit, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + def call(self, prompt: str) -> dict: + return {"success": True, "response": "[]", "usage": {}} + + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) + assert main(args(human_audit, html, "--live", "--approve-live")) == 0 + run = directory(tmp_path) + rows = read_review(run / "review.csv") + for row in rows: + row.update(classification="llm_eligible") + write_review(run / "review.csv", rows) + (run / "artifacts" / filename).rename(run / filename) + before = {path.relative_to(run): path.read_bytes() for path in run.rglob("*") if path.is_file()} + with pytest.raises(SystemExit) as error: + main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) + assert error.value.code == 2 + assert str(run / "artifacts" / filename) in capsys.readouterr().err + assert {path.relative_to(run): path.read_bytes() for path in run.rglob("*") if path.is_file()} == before + + +@pytest.mark.parametrize("filename,field", [ + ("run.json", "run_id"), + ("artifacts/human-findings.json", "human_audit_sha256"), + ("normalized-llm-findings.json", "run_id"), + ("normalized-llm-findings.json", "source"), + ("normalized-llm-findings.json", "page_url"), + ("normalized-programmatic-findings.json", "run_id"), + ("normalized-programmatic-findings.json", "source"), + ("normalized-programmatic-findings.json", "page_url"), +]) +def test_report_rejects_evidence_from_another_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], filename: str, field: str, +) -> None: + """Moving supporting evidence preserves all provenance validation.""" + human_audit, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + class FakeClient: + def call(self, prompt: str) -> dict: + return {"success": True, "response": '[{"problem": "Unclear link"}]', "usage": {}} + + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) + assert main(args(human_audit, html, "--live", "--approve-live")) == 0 + run = directory(tmp_path) + path = run / filename + evidence = json.loads(path.read_text()) + record = evidence[0] if isinstance(evidence, list) else evidence + record[field] = "unrelated" + path.write_text(json.dumps(evidence)) + with pytest.raises(SystemExit) as error: + main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) + assert error.value.code == 2 + message = capsys.readouterr().err + if field == "human_audit_sha256": + assert "identity differs" in message + else: + assert "do not belong" in message or "does not belong" in message + assert not (run / "report.md").exists() diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index c52d236..09fe2af 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -91,23 +91,25 @@ def _run(args: argparse.Namespace) -> int: raise ValueError(f"Benchmark HTML not found: {args.html}") human_audit = import_homepage_human_findings(args.human_audit, args.source_url) run = _new_run_directory() + artifacts = run / "artifacts" + artifacts.mkdir() shutil.copyfile(args.html, run / "snapshot.html") - shutil.copyfile(DEFAULT_SCHEDULE, run / "pricing.json") + shutil.copyfile(DEFAULT_SCHEDULE, artifacts / "pricing.json") snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() - write_json(run / "human-findings.json", human_audit.to_dict()) + write_json(artifacts / "human-findings.json", human_audit.to_dict()) production = run_pipeline( html_path=str(run / "snapshot.html"), - output_dir=run, + output_dir=artifacts, api_key=None, model=MODEL, dry_run=True, include_summaries=False, request_config=REQUEST_CONFIG, ) - (run / "manifest.json").unlink() - raw_path = run / "programmatic_findings.json" + (artifacts / "manifest.json").unlink() + raw_path = artifacts / "programmatic_findings.json" raw_programmatic = json.loads(raw_path.read_text()) - raw_path.rename(run / "raw-programmatic-findings.json") + raw_path.rename(artifacts / "raw-programmatic-findings.json") findings = normalize_programmatic_findings( raw_programmatic, run_id=run.name, @@ -118,7 +120,7 @@ def _run(args: argparse.Namespace) -> int: [asdict(item) for item in findings], ) create_review(run / "review.csv", list(human_audit.to_dict()["human_findings"])) - schedule = PriceSchedule.load(run / "pricing.json") + schedule = PriceSchedule.load(artifacts / "pricing.json") manifest = { "run_id": run.name, "source_url": args.source_url, @@ -190,12 +192,13 @@ def _report(args: argparse.Namespace) -> int: if not args.reviewer.strip(): raise ValueError("A reviewer description is required") run = args.run_dir.resolve() + artifacts = run / "artifacts" manifest = json.loads((run / "run.json").read_text()) if manifest["run_id"] != run.name: raise ValueError("Run evidence does not belong to this directory") if not manifest["complete"]: raise ValueError("Incomplete Evaluation runs cannot produce a final report") - human_audit = json.loads((run / "human-findings.json").read_text()) + human_audit = json.loads((artifacts / "human-findings.json").read_text()) if human_audit["human_audit_sha256"] != manifest["human_audit_sha256"]: raise ValueError("Human audit identity differs from the run") llm = json.loads((run / "normalized-llm-findings.json").read_text()) @@ -212,7 +215,7 @@ def _report(args: argparse.Namespace) -> int: rows = reviewed_rows( run / "review.csv", human_audit["human_findings"], llm, programmatic ) - schedule = PriceSchedule.load(run / "pricing.json") + schedule = PriceSchedule.load(artifacts / "pricing.json") write_report( run / "report.md", manifest, rows, llm, programmatic, args.reviewer, str(schedule.data["published_at"]), diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 3645e66..6e459bd 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -60,7 +60,8 @@ def positive_cost(value: str) -> Decimal: def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: """Make each paid request once, saving reported usage and responses as received.""" - schedule = PriceSchedule.load(run / "pricing.json") + artifacts = run / "artifacts" + schedule = PriceSchedule.load(artifacts / "pricing.json") client = LLMRequestClient( api_key=api_key, request_config=REQUEST_CONFIG, @@ -79,7 +80,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: complete, reason = False, "cost guardrail exhausted" break name = prompt["name"] - prompt_text = json.loads((run / "prompts" / f"{name}.json").read_text())[ + prompt_text = json.loads((artifacts / "prompts" / f"{name}.json").read_text())[ "prompt_text" ] result: dict[str, Any] @@ -117,7 +118,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: incomplete_reason=reason, format_failures=format_failures, ) - write_json(run / "raw-llm-responses.json", responses) + write_json(artifacts / "raw-llm-responses.json", responses) write_json(run / "normalized-llm-findings.json", findings) write_json(run / "run.json", manifest) if not complete: From 49b50c522b70f788933271dbac4388451055ec58 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 10:24:16 -0400 Subject: [PATCH 054/126] docs(tracker): resolve run directory layout spec --- .scratch/run-directory-layout/spec.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.scratch/run-directory-layout/spec.md b/.scratch/run-directory-layout/spec.md index 0bbe805..d432583 100644 --- a/.scratch/run-directory-layout/spec.md +++ b/.scratch/run-directory-layout/spec.md @@ -1,6 +1,6 @@ # Organize the Evaluation run directory for review and results -Status: ready-for-agent +Status: resolved ## Problem Statement @@ -106,6 +106,16 @@ This is a clean break. New runs use only the new layout, and the updated report - Run the evaluation CLI lifecycle suite and the repository's applicable existing checks. Reuse existing shared-pipeline checks to confirm its ordinary output contract remains unchanged. - No paid live verification is part of this spec. +## Acceptance Criteria + +- [x] Complete runs expose exactly the agreed review files and `artifacts/` at the root; successful reporting adds only `report.md`. +- [x] Supporting evidence retains its filenames under `artifacts/`, with prompts and payloads nested inside it. +- [x] Preview, declined, stopped, and malformed-response cases preserve artifact availability, approval, completion, usage, and per-prompt persistence behavior. +- [x] Run identity, finding provenance, Human audit identity, review validation, report content, and hand-calculated detection rates remain correct. +- [x] Reporting reads only the new supporting locations and neither falls back to root files nor reorganizes existing evidence. +- [x] The shared pipeline's ordinary output layout remains unchanged outside evaluation. +- [x] README workflow, artifact references, review instructions, and the manual-reorganization requirement match the implemented layout. + ## Out of Scope - Generated reviewer guides, readable finding references, or replacement review interfaces. @@ -135,3 +145,7 @@ The plan preserves the existing decisions to separate human review from determin - Q4: The operator chose one supporting artifact directory with existing filenames. - Q5: The operator chose the new layout only, leaving existing directories untouched. - Q6: The operator confirmed the complete plan and requested publication through the to-spec skill. + +- Implementation complete in commit `1bc12d99` (`feat(evaluation)!: organize supporting run artifacts`). +- Verification: all 47 repository tests passed, including 18 evaluation CLI lifecycle cases. The ordinary shared-pipeline dry run retained its output contract with 10 prompts, 14 Programmatic findings, and zero provider tokens. Lockfile, dependency export, entry-point imports, compilation, and diff checks passed. The repository has no configured type checker. No paid requests or private-run mutations were performed. +- Review: independent Standards and Spec reviews each reported zero findings. All acceptance criteria above are demonstrated by lifecycle tests, the shared-pipeline check, or review. Resolved after the implementation commit and both reviews. From bd1d9f682ecedbb0abad09c369fa549607907eb1 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 11:37:42 -0400 Subject: [PATCH 055/126] feat(evaluation): select Haiku, Opus, or Sonnet presets --- .scratch/evaluation-model-selection/spec.md | 108 ++++++++++++++ CONTEXT.md | 2 +- README.md | 29 +++- processing_scripts/llm_client/audit.py | 32 +++- tests/test_audit_request.py | 23 ++- tests/test_evaluation_cli.py | 155 +++++++++++++++++++- vision_aid/evaluation/cli.py | 46 ++++-- vision_aid/evaluation/llm_audit.py | 65 +++++--- vision_aid/evaluation/pricing.json | 16 +- vision_aid/evaluation/pricing.py | 7 + vision_aid/evaluation/review.py | 4 +- 11 files changed, 430 insertions(+), 57 deletions(-) create mode 100644 .scratch/evaluation-model-selection/spec.md diff --git a/.scratch/evaluation-model-selection/spec.md b/.scratch/evaluation-model-selection/spec.md new file mode 100644 index 0000000..5956d3d --- /dev/null +++ b/.scratch/evaluation-model-selection/spec.md @@ -0,0 +1,108 @@ +# Select a model for an Evaluation run + +Design decisions settled and implementation authorized on October 1, 2026. + +## Settled decisions + +- Support a hardcoded set of three models through Anthropic's direct API: Haiku 4.5, Opus 5.5, and Sonnet 5.5. +- Select one model per Evaluation run through `--model`; retain Haiku 4.5 as the default. +- Each Evaluation run holds its selected model fixed throughout. A command that evaluates several models is outside this feature. +- Accept only the three canonical API IDs below. Reject unsupported IDs, including convenience aliases, with an error listing the supported choices. +- Use fixed request presets with no operator overrides for thinking, effort, or output token limits. +- Preserve Haiku's existing thinking budget. Use adaptive thinking with explicit `high` effort for both Opus and Sonnet. +- Keep the existing 24,192 output-token cap for all three models. This cap includes thinking and final text; adaptive thinking does not reserve a fixed number of final-text tokens. +- Preserve the positive cost guardrail checked between requests. A request underway can take the final Estimated run cost above the limit. +- Add versioned public prices for the new models, and show the selected model's settings and rates before Live-run approval. + +This changes the Haiku-only scope in [the POC reduction spec](../poc-reduction/spec.md). Human review and scoring continue to follow [ADR 0001](../../docs/adr/0001-separate-review-from-scoring.md) and [ADR 0002](../../docs/adr/0002-rank-models-by-llm-detection-rate.md). + +## Model presets + +| Model | Accepted `--model` value | Thinking | Effort | Output-token cap | +| --- | --- | --- | --- | --- | +| Haiku 4.5, default | `claude-haiku-4-5-20251001` | Enabled, 16,000-token budget | Omitted | 24,192 | +| Opus 5.5 | `claude-opus-5-5` | Adaptive, no manual budget | `high` | 24,192 | +| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive, no manual budget | `high` | 24,192 | + +All presets omit temperature, disable summaries, and use standard Anthropic Messages requests. These exact IDs identify fixed model snapshots. The dateless Opus and Sonnet IDs do not require invented date suffixes. + +Example selection: + +```bash +uv run visionaid-evaluate run \ + --html .model-evaluation/pristine-home.html \ + --source-url https://pristineai.com/ \ + --model claude-opus-5-5 \ + --max-cost-usd 1.00 +``` + +This example prepares a preview. The existing `--live` and Live-run approval flow enables paid execution. + +## Run behavior and evidence + +- Resolve the selected preset once and use it consistently for prompt preparation, paid execution, normalization, cost estimates, and saved run metadata. Execution must not silently fall back to global Haiku settings. +- Record the canonical model ID, provider and endpoint, thinking mode, manual budget when applicable, effort when applicable, output cap, omitted temperature, and disabled summaries in `run.json`. Retain provider-returned model identity in raw response evidence. +- Display the selected preset, applicable token rates, versioned price-schedule identity, prompt count, and positive cost guardrail before asking for Live-run approval. +- Preserve key-free previews, no requests after declined approval, explicit authorization for paid execution, sequential requests, disabled retries, saved partial evidence, and stopping on request failure or accumulated cost exhaustion. +- A model that is unavailable to the operator's account fails with retained evidence under the existing request-failure behavior. Do not substitute another model. +- Continue to generate reports from saved run evidence, including previously completed Haiku runs whose metadata predates model selection. Human review and detection-rate scoring do not change. +- Preserve the current handling of malformed responses and output-limit stop reasons; changing completion semantics is outside this feature. + +## Versioned pricing + +Standard public rates checked October 1, 2026, in USD per million tokens: + +| Model | Input | Output | Cache read | 5-minute cache write | +| --- | ---: | ---: | ---: | ---: | +| Haiku 4.5 | 1 | 5 | 0.10 | 1.25 | +| Opus 5.5 | 4 | 20 | 0.20 | 5 | +| Sonnet 5.5 | 2 | 10 | 0.20 | 2.50 | + +Use exact rates from the recorded schedule, including Opus's cache-read rate rather than assuming a common multiplier. Keep the schedule copy and its content identity with each run. Existing run evidence retains its original schedule. Record the rate source and capture date accurately; a capture date is not necessarily the publisher's publication date. + +This feature uses standard pricing and does not add fast mode, batch execution, residency options, or new caching behavior. + +## Implementation constraints + +- Keep model selection and the three-model restriction in the evaluator. The shared client remains usable by existing production callers with their current defaults. +- Add explicit support for adaptive thinking and Anthropic effort in the shared request configuration. For the new models, send adaptive thinking and `output_config` with `effort` set to `high`; omit `budget_tokens`. +- Stream all three evaluation presets. The Anthropic Python SDK requires streaming above 21,333 output tokens, and all presets retain a 24,192 cap. Streaming must not depend only on the presence of a manual thinking budget. +- Continue extracting final text separately from thinking content and preserving reported usage and failures in raw evidence. +- No model-discovery or dynamic-pricing subsystem is required. Adding another model later requires an explicit preset and versioned prices. +- Update evaluator help and README instructions to describe selection, the default, settings, and the existing between-request cost limit. + +## Acceptance criteria + +- [ ] Omitted `--model` selects the existing Haiku preset; each of the three exact IDs is accepted in preview and live paths. +- [ ] An unsupported ID is rejected before creating a run or making provider requests, and the error lists supported choices. +- [ ] Every selected model uses its prescribed request settings and appears consistently in run metadata, request evidence, LLM findings, cost calculations, and the final report. +- [ ] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `high` effort for both new models, omitted temperature, streaming, and final-text extraction. +- [ ] Cost tests verify each model's token categories, including Opus cache-read pricing, using saved versioned prices. +- [ ] Preview, declined approval, failure, and cost exhaustion retain existing lifecycle behavior for selected models. +- [ ] Previously completed Haiku evidence still produces a report. +- [ ] Shared-client regression checks confirm existing production caller defaults remain intact. +- [ ] CLI help and README show model selection and explain that the guardrail can be exceeded by a request underway. + +Verification uses mocked providers and local fixtures; this feature does not require a paid verification run. + +## Remaining design decisions + +None. + +## Starting code facts + +- `vision_aid/evaluation/llm_audit.py` pins Haiku, its request settings, and the model used for normalization and cost estimation. +- `vision_aid/evaluation/cli.py` has no model selection option and uses those fixed settings for preparation, run metadata, and Live-run approval. +- The versioned price schedule contains only the pinned Haiku ID. Cost estimation requires an exact model match. +- The shared request client supports manual extended thinking but has no adaptive thinking or Anthropic effort settings. +- Reports already read the model from saved run metadata. + +## Sources + +- [Anthropic model overview](https://platform.claude.com/docs/en/models/overview), checked October 1, 2026. +- [Anthropic pricing](https://platform.claude.com/docs/en/about-claude/pricing), checked October 1, 2026. +- [Anthropic thinking configuration and streaming constraints](https://platform.claude.com/docs/en/build-with-claude/thinking), checked October 1, 2026. +- [Anthropic effort configuration](https://platform.claude.com/docs/en/build-with-claude/effort), checked October 1, 2026. +- [Anthropic model IDs and versioning](https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions), checked October 1, 2026. + +Opus defaults to medium effort and Sonnet to high. Explicit high effort for both is the agreed evaluation preset. Equal effort labels and output caps do not imply equal computation across models. diff --git a/CONTEXT.md b/CONTEXT.md index 551f88b..c0e8d79 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -17,7 +17,7 @@ The saved HTML of the audited page, identified by its source URL and checksum. _Avoid_: Live page. **Evaluation run**: -One audit with a fixed model, together with its evidence, the human decisions, its completion status, and its report. +One audit with one selected model held fixed throughout, together with its evidence, the human decisions, its completion status, and its report. _Avoid_: Execution plan, report bundle. **LLM**: diff --git a/README.md b/README.md index eecbadd..17746b3 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,25 @@ GEMINI_API_KEY=AIza... ## Private Model Evaluation +### Choose an evaluation model + +Each Evaluation run uses one fixed model through Anthropic's direct API. +Choose it with `--model`; omitting the flag selects Haiku 4.5. The evaluator +accepts only these exact IDs: + +| Model | `--model` value | Thinking | Effort | +| --- | --- | --- | --- | +| Haiku 4.5, default | `claude-haiku-4-5-20251001` | 16,000-token budget | Omitted | +| Opus 5.5 | `claude-opus-5-5` | Adaptive | High | +| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | High | + +For example, add `--model claude-opus-5-5` to the run command below to evaluate +Opus. Friendly names and aliases are rejected. Thinking settings are fixed +presets, and all three models use a 24,192-token output cap that includes +thinking and final text. Adaptive thinking does not reserve a fixed number of +tokens for final text. Temperature is omitted and summaries are disabled. +The selected model and settings are saved with each run. + ### Get a final report with the supplied Human audit From the repository root, use the supplied `Pristine Accessibility Defect Report.xlsx` @@ -275,8 +294,10 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report --live ``` - The CLI shows Haiku's 16,000 thinking tokens, 24,192 output token cap, - prompt count, pricing, snapshot checksum, and limit. Type `yes` to approve + The CLI shows the selected model's thinking settings, effort, output token cap, + prompt count, token rates, pricing version, snapshot checksum, and limit. + The cost guardrail is checked between requests; a request underway can exceed it. + Type `yes` to approve paid requests; `--approve-live` skips the prompt. It copies the snapshot, imports Human findings, runs checks, and saves responses and usage. @@ -337,7 +358,7 @@ compatibility fallback or automatic migration. | `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | | `report.md` | Scored results generated after successful reporting. | | `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | -| `artifacts/pricing.json` | The versioned model rates used to estimate this run's cost from reported usage. | +| `artifacts/pricing.json` | The versioned public model rates and capture date used to estimate this run's cost from reported usage. Older runs retain their original price schedule. | | `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | | `artifacts/raw-programmatic-findings.json` | Original output from the Programmatic checks. | | `artifacts/prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | @@ -421,7 +442,7 @@ uv run visionaid-evaluate run --html .model-evaluation/pristine-home.html \ This needs no API key or approval. It creates an incomplete run with `artifacts/raw-programmatic-findings.json`, `normalized-programmatic-findings.json`, LLM prompts, and `review.csv`. Use the normalized file's `finding_id` values -for programmatic matches. It makes no Haiku requests and cannot produce a +for programmatic matches. It makes no model requests and cannot produce a final report. `--max-cost-usd` must be positive but is not spent without `--live`. Start a new live run at step 3 to finish the evaluation. diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index ebc7315..6869882 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -36,6 +36,7 @@ class AuditRequestConfig: temperature: float | None = 0.1 max_output_tokens: int = 8192 thinking_budget_tokens: int | None = None + adaptive_thinking: bool = False def __post_init__(self) -> None: """Reject configuration values that cannot form an audit request.""" @@ -44,6 +45,16 @@ def __post_init__(self) -> None: if self.max_output_tokens <= 0: raise ValueError("max_output_tokens must be greater than zero") + if self.adaptive_thinking: + if is_openai_model(self.model) or is_gemini_model(self.model): + raise ValueError("adaptive_thinking requires Anthropic") + if self.thinking_budget_tokens is not None: + raise ValueError("adaptive thinking cannot have a manual budget") + if self.temperature is not None: + raise ValueError("temperature must be omitted with thinking") + if self.reasoning_effort not in (None, "low", "medium", "high", "xhigh", "max"): + raise ValueError("unsupported Anthropic reasoning effort") + if self.thinking_budget_tokens is not None: if is_openai_model(self.model) or is_gemini_model(self.model): raise ValueError("thinking_budget_tokens requires Anthropic") @@ -56,6 +67,7 @@ def as_metadata(self) -> dict[str, Any]: """Return the requested configuration as serializable metadata.""" omitted = [] if self.temperature is not None else ["temperature"] return { + **({"thinking": {"type": "adaptive"}} if self.adaptive_thinking else {}), **( { "thinking": { @@ -191,11 +203,16 @@ def __init__( self._is_openai = is_openai_model(self.model) self._is_gemini = is_gemini_model(self.model) - if request_config.reasoning_effort is not None and not self._is_openai: + if ( + request_config.reasoning_effort is not None + and not self._is_openai + and not request_config.adaptive_thinking + ): raise ValueError( - "reasoning_effort is only supported for OpenAI audit requests" + "reasoning_effort requires OpenAI or Anthropic adaptive thinking" ) + self._client: Any if provider_client is not None: self._client = provider_client elif self._is_openai: @@ -293,6 +310,17 @@ def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: "type": "enabled", "budget_tokens": self.request_config.thinking_budget_tokens, } + elif self.request_config.adaptive_thinking: + request_kwargs["thinking"] = {"type": "adaptive"} + if self.request_config.reasoning_effort is not None: + request_kwargs["output_config"] = { + "effort": self.request_config.reasoning_effort, + } + + if ( + self.request_config.thinking_budget_tokens is not None + or self.request_config.adaptive_thinking + ): message = self._stream_anthropic(request_kwargs) else: message = self._client.messages.create(**request_kwargs) diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index d412a99..dc99246 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -262,7 +262,12 @@ def test_pipeline_persists_explicit_request_configuration(self): @pytest.mark.parametrize( "parts,stop", [(["[", "]"], "end_turn"), ([], "end_turn"), (["["], "max_tokens")] ) -def test_haiku_thinking_stream_exposes_only_final_text(parts, stop): +@pytest.mark.parametrize("model,thinking,effort", [ + ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None), + ("claude-opus-5-5", {"type": "adaptive"}, "high"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "high"), +]) +def test_thinking_stream_exposes_only_final_text(parts, stop, model, thinking, effort): """Only final text and inclusive billed output leave a thinking stream.""" from contextlib import nullcontext @@ -276,7 +281,7 @@ def event(kind, **values): "message_start", message=SimpleNamespace( id="msg-thinking", - model="claude-haiku-4-5-20251001", + model=model, type="message", usage=SimpleNamespace(input_tokens=100, output_tokens=1), ), @@ -309,22 +314,28 @@ def event(kind, **values): "unused", provider_client=fake, request_config=AuditRequestConfig( - model="claude-haiku-4-5-20251001", + model=model, temperature=None, - thinking_budget_tokens=16000, + thinking_budget_tokens=thinking.get("budget_tokens"), + adaptive_thinking=thinking["type"] == "adaptive", + reasoning_effort=effort, max_output_tokens=24192, ), ).call("audit") assert fake.messages.requests == [ { - "model": "claude-haiku-4-5-20251001", + "model": model, "max_tokens": 24192, "messages": [{"role": "user", "content": "audit"}], - "thinking": {"type": "enabled", "budget_tokens": 16000}, + "thinking": thinking, + **({"output_config": {"effort": effort}} if effort else {}), "stream": True, } ] assert result["success"] + assert result["model"] == model + assert result["request"]["thinking"] == thinking + assert result["request"]["reasoning_effort"] == effort assert result["response"] == "".join(parts) assert result["stop_reason"] == stop assert result["usage"]["output_tokens"] == 16002 diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index 671b34d..a59498a 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -1,9 +1,11 @@ """Run, review, and report through the CLI and its saved artifacts.""" import csv +from contextlib import nullcontext import hashlib import json from pathlib import Path +from types import SimpleNamespace import openpyxl import pytest @@ -88,6 +90,129 @@ def write_review(path: Path, rows: list[dict[str, str]]) -> None: writer.writerows(rows) +@pytest.mark.parametrize("model,thinking,effort,input_rate,output_rate", [ + ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "1", "5"), + ("claude-opus-5-5", {"type": "adaptive"}, "high", "4", "20"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "high", "2", "10"), +]) +def test_selected_model_preview_records_preset_without_provider_access( + tmp_path, monkeypatch, capsys, model, thinking, effort, input_rate, output_rate, +): + """A key-free preview records and displays the selected preset and prices.""" + human_audit, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + + assert main(args(human_audit, html, "--model", model)) == 0 + run = directory(tmp_path) + manifest = json.loads((run / "run.json").read_text()) + config = manifest["configuration"] + assert config["model"] == model + assert config["thinking"] == thinking + assert config["reasoning_effort"] == effort + assert config["max_output_tokens"] == 24192 + assert config["temperature"] == "omitted" + assert config["summaries"] == "disabled" + assert config["provider"] == "anthropic" and config["endpoint"] == "messages" + schedule = json.loads((run / "artifacts" / "pricing.json").read_text()) + assert schedule["models"][model]["input_per_million_usd"] == input_rate + assert schedule["models"][model]["output_per_million_usd"] == output_rate + assert manifest["pricing_sha256"] == hashlib.sha256((run / "artifacts" / "pricing.json").read_bytes()).hexdigest() + assert not (run / "artifacts" / "raw-llm-responses.json").exists() + output = capsys.readouterr().out + assert model in output and "24192" in output + assert f"input ${input_rate}; output ${output_rate}" in output + assert "16000" in output if effort is None else "adaptive; effort: high" in output + assert "request underway can exceed" in output + + +@pytest.mark.parametrize("model", ["claude-haiku-4-5", "claude-unknown", "gpt-4o", "opus"]) +def test_unsupported_model_is_rejected_without_run_or_provider_access( + tmp_path, monkeypatch, capsys, model, +): + """Unsupported IDs and aliases fail before creating evidence or buying requests.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + with pytest.raises(SystemExit) as error: + main(args(tmp_path / "missing.xlsx", tmp_path / "missing.html", "--model", model, "--live", "--approve-live")) + assert error.value.code == 2 + message = capsys.readouterr().err + assert "invalid choice" in message + assert all(supported in message for supported in ( + "claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5", + )) + assert not (tmp_path / ".model-evaluation").exists() + + +@pytest.mark.parametrize("model,thinking,effort,cost", [ + ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "0.006525"), + ("claude-opus-5-5", {"type": "adaptive"}, "high", "0.0258"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "high", "0.01305"), +]) +def test_selected_model_executes_and_reports_with_saved_prices( + tmp_path, monkeypatch, model, thinking, effort, cost, +): + """The real CLI and request client retain selected-model evidence and costs.""" + human_audit, html = inputs(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + requests = [] + client_options = [] + + def create(**kwargs): + requests.append(kwargs) + assert kwargs["model"] == model + assert kwargs["thinking"] == thinking + assert kwargs["max_tokens"] == 24192 and kwargs["stream"] is True + assert "temperature" not in kwargs + if effort: + assert kwargs["output_config"] == {"effort": "high"} + else: + assert "output_config" not in kwargs + return nullcontext(iter([ + SimpleNamespace(type="message_start", message=SimpleNamespace( + id="synthetic-response", model=model, type="message", + usage=SimpleNamespace(input_tokens=1000, output_tokens=1, + cache_read_input_tokens=500, cache_creation_input_tokens=100), + )), + SimpleNamespace(type="content_block_start", content_block=SimpleNamespace(type="thinking", thinking="SECRET")), + SimpleNamespace(type="content_block_delta", delta=SimpleNamespace(type="thinking_delta", thinking="SECRET")), + SimpleNamespace(type="content_block_start", content_block=SimpleNamespace(type="text", text='[{"problem": "Unclear link"}]')), + SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), usage=SimpleNamespace(output_tokens=200)), + SimpleNamespace(type="message_stop"), + ])) + + def client(**options): + client_options.append(options) + return SimpleNamespace(messages=SimpleNamespace(create=create)) + + monkeypatch.setattr("anthropic.Anthropic", client) + assert main(args(human_audit, html, "--model", model, "--max-cost-usd", "1", "--live", "--approve-live")) == 0 + assert client_options == [{"api_key": "synthetic", "max_retries": 0}] + assert len(requests) == 3 + run = directory(tmp_path) + manifest = json.loads((run / "run.json").read_text()) + assert manifest["configuration"]["model"] == model + assert manifest["configuration"]["thinking"] == thinking + assert manifest["configuration"]["reasoning_effort"] == effort + assert manifest["estimated_cost_usd"] == cost + findings = json.loads((run / "normalized-llm-findings.json").read_text()) + assert len(findings) == 3 and all(finding["model"] == model for finding in findings) + raw = json.loads((run / "artifacts" / "raw-llm-responses.json").read_text()) + assert all(item["result"]["provider"]["response_model"] == model for item in raw) + assert all(item["result"]["request"]["thinking"] == thinking for item in raw) + assert "SECRET" not in json.dumps(raw) + rows = read_review(run / "review.csv") + for row in rows: + row.update(classification="llm_eligible") + write_review(run / "review.csv", rows) + assert main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) == 0 + report = (run / "report.md").read_text() + assert model in report + assert "published prices captured on 2026-10-01" in report + + def test_preview_preserves_human_findings_and_never_calls_provider( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: @@ -259,6 +384,24 @@ def client(**options: object) -> FakeClient: with pytest.raises(SystemExit): main(["report", "--run-dir", str(run), "--reviewer", " "]) write_review(review, list(reversed(rows))) + # Reproduce saved evidence from before selectable model presets existed. + old_manifest = json.loads((run / "run.json").read_text()) + old_manifest["configuration"].pop("thinking") + old_manifest["configuration"].pop("reasoning_effort") + old_pricing = { + "models": {"claude-haiku-4-5-20251001": { + "input_per_million_usd": "1", "output_per_million_usd": "5", + "cached_input_per_million_usd": "0.10", "cache_creation_input_per_million_usd": "1.25", + }}, + "published_at": "2026-09-22", + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "version": "anthropic-public-2026-09-22", + } + pricing_path = run / "artifacts" / "pricing.json" + pricing_path.write_text(json.dumps(old_pricing)) + old_manifest["pricing_version"] = old_pricing["version"] + old_manifest["pricing_sha256"] = hashlib.sha256(pricing_path.read_bytes()).hexdigest() + (run / "run.json").write_text(json.dumps(old_manifest)) assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 assert {path.name for path in run.iterdir()} == root_entries | {"report.md"} report = (run / "report.md").read_text() @@ -463,8 +606,9 @@ def call(self, prompt: str) -> dict: assert "Of all 3 Human findings, either part of the tool caught 0 (0.0%)" in (run / "report.md").read_text() +@pytest.mark.parametrize("model", ["claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5"]) def test_live_execution_needs_explicit_approval( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, model: str, ) -> None: """A noninteractive live flag without approval makes no paid request.""" human_audit, html = inputs(tmp_path) @@ -480,7 +624,7 @@ def isatty(self) -> bool: return False monkeypatch.setattr("sys.stdin", DeclinedInput()) - assert main(args(human_audit, html, "--live")) == 0 + assert main(args(human_audit, html, "--model", model, "--live")) == 0 run = directory(tmp_path) manifest = json.loads((run / "run.json").read_text()) assert manifest["complete"] is False @@ -494,8 +638,9 @@ def isatty(self) -> bool: @pytest.mark.parametrize("failure", [True, False]) +@pytest.mark.parametrize("model", ["claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5"]) def test_failed_or_over_budget_runs_keep_usage_and_cannot_report( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool, model: str, ) -> None: """Failure and budget exhaustion stop after one request and remain inspectable.""" human_audit, html = inputs(tmp_path) @@ -517,7 +662,7 @@ def call(self, prompt: str) -> dict: fake = FakeClient() monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: fake) - assert main(args(human_audit, html, "--live", "--approve-live")) == 1 + assert main(args(human_audit, html, "--model", model, "--live", "--approve-live")) == 1 run = directory(tmp_path) manifest = json.loads((run / "run.json").read_text()) assert fake.calls == 1 @@ -533,6 +678,8 @@ def call(self, prompt: str) -> dict: assert len(findings) == (0 if failure else 1) assert all(item["run_id"] == run.name for item in findings) assert manifest["usage"]["input_tokens"] == 100000 + assert manifest["configuration"]["model"] == model + assert all(item["model"] == model for item in findings) assert float(manifest["estimated_cost_usd"]) > 0 assert len(json.loads((run / "artifacts" / "raw-llm-responses.json").read_text())) == 1 with pytest.raises(SystemExit): diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 09fe2af..25a1a51 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -1,4 +1,4 @@ -"""Run one private Haiku evaluation, then score a human-edited CSV.""" +"""Run one private fixed-model evaluation, then score a human-edited CSV.""" from __future__ import annotations @@ -19,9 +19,9 @@ from entry_points.run_pipeline import run_pipeline from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( - CONFIGURATION, - MODEL, - REQUEST_CONFIG, + DEFAULT_MODEL, + MODEL_PRESETS, + evaluation_configuration, execute, positive_cost, write_json, @@ -39,7 +39,11 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="visionaid-evaluate") commands = parser.add_subparsers(dest="command", required=True) run = commands.add_parser( - "run", help="preview or execute the fixed Haiku experiment" + "run", help="preview or execute an Evaluation run with one selected model" + ) + run.add_argument( + "--model", choices=MODEL_PRESETS, default=DEFAULT_MODEL, + help=f"fixed model preset (default: {DEFAULT_MODEL})", ) run.add_argument( "--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT, help="Human audit" @@ -84,6 +88,8 @@ def _new_run_directory() -> Path: def _run(args: argparse.Namespace) -> int: """Prepare a fresh run and execute only after explicit approval.""" + request_config = MODEL_PRESETS[args.model] + configuration = evaluation_configuration(request_config) limit = positive_cost(args.max_cost_usd) if args.approve_live and not args.live: raise ValueError("--approve-live requires --live") @@ -101,10 +107,10 @@ def _run(args: argparse.Namespace) -> int: html_path=str(run / "snapshot.html"), output_dir=artifacts, api_key=None, - model=MODEL, + model=request_config.model, dry_run=True, include_summaries=False, - request_config=REQUEST_CONFIG, + request_config=request_config, ) (artifacts / "manifest.json").unlink() raw_path = artifacts / "programmatic_findings.json" @@ -127,7 +133,7 @@ def _run(args: argparse.Namespace) -> int: "human_audit_filename": human_audit.human_audit_filename, "human_audit_sha256": human_audit.human_audit_sha256, "snapshot_sha256": snapshot_hash, - "configuration": CONFIGURATION, + "configuration": configuration, "pricing_version": schedule.version, "pricing_sha256": schedule.identity, "max_cost_usd": str(limit), @@ -154,9 +160,11 @@ def _run(args: argparse.Namespace) -> int: write_json(run / "run.json", manifest) print(f"Evaluation run: {run}") print( - f"Model: {MODEL}; endpoint: Messages; " - f"thinking: {CONFIGURATION['thinking_budget_tokens']}; " - f"output cap: {CONFIGURATION['max_output_tokens']}" + f"Model: {request_config.model}; endpoint: Messages; " + f"thinking: {configuration['thinking']['type']}; " + f"effort: {request_config.reasoning_effort or 'omitted'}; " + f"thinking budget: {request_config.thinking_budget_tokens or 'none'}; " + f"output cap: {request_config.max_output_tokens}" ) print( "Temperature: omitted; summaries: disabled; " @@ -167,7 +175,17 @@ def _run(args: argparse.Namespace) -> int: f"Human findings: {len(human_audit.human_findings)}" ) print(f"Pricing: {schedule.version} ({schedule.identity})") - print(f"Cost guardrail: ${limit}; checked before each request") + rates = schedule.data["models"][request_config.model] + print( + f"Rates per million tokens (USD): input ${rates['input_per_million_usd']}; " + f"output ${rates['output_per_million_usd']}; " + f"cache read ${rates['cached_input_per_million_usd']}; " + f"5-minute cache write ${rates['cache_creation_input_per_million_usd']}" + ) + print( + f"Cost guardrail: ${limit}; checked before each request; " + "a request underway can exceed the limit" + ) print(f"Review CSV: {run / 'review.csv'}") if not args.live: print("Preview complete; no paid requests made.") @@ -182,7 +200,7 @@ def _run(args: argparse.Namespace) -> int: ): print("Live execution declined; no paid requests made.") return 0 - complete = execute(run, manifest, api_key) + complete = execute(run, manifest, api_key, request_config) print(f"Run {'complete' if complete else 'incomplete'}: {run}") return 0 if complete else 1 @@ -218,7 +236,7 @@ def _report(args: argparse.Namespace) -> int: schedule = PriceSchedule.load(artifacts / "pricing.json") write_report( run / "report.md", manifest, rows, llm, programmatic, - args.reviewer, str(schedule.data["published_at"]), + args.reviewer, schedule.price_description, ) print(f"Markdown report: {run / 'report.md'}") return 0 diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 6e459bd..ee091f0 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -1,4 +1,4 @@ -"""Fixed Haiku request configuration and sequential paid execution.""" +"""Fixed evaluation model presets and sequential paid execution.""" from __future__ import annotations @@ -15,21 +15,24 @@ from vision_aid.evaluation.normalization import normalize_prompt_response from vision_aid.evaluation.pricing import PriceSchedule -MODEL = "claude-haiku-4-5-20251001" -REQUEST_CONFIG = LLMRequestConfig( - model=MODEL, - thinking_budget_tokens=16000, - max_output_tokens=24192, - temperature=None, -) -CONFIGURATION = { - "model": MODEL, - "provider": "anthropic", - "endpoint": "messages", - "thinking_budget_tokens": REQUEST_CONFIG.thinking_budget_tokens, - "max_output_tokens": REQUEST_CONFIG.max_output_tokens, - "temperature": "omitted", - "summaries": "disabled", +DEFAULT_MODEL = "claude-haiku-4-5-20251001" +MODEL_PRESETS = { + DEFAULT_MODEL: LLMRequestConfig( + model=DEFAULT_MODEL, + thinking_budget_tokens=16000, + max_output_tokens=24192, + temperature=None, + ), + **{ + model: LLMRequestConfig( + model=model, + adaptive_thinking=True, + reasoning_effort="high", + max_output_tokens=24192, + temperature=None, + ) + for model in ("claude-opus-5-5", "claude-sonnet-5-5") + }, } USAGE_KEYS = ( "input_tokens", @@ -39,6 +42,21 @@ ) +def evaluation_configuration(config: LLMRequestConfig) -> dict[str, Any]: + """Record the selected preset and evaluation-only request settings.""" + return { + "model": config.model, + "provider": "anthropic", + "endpoint": "messages", + "thinking": config.as_metadata()["thinking"], + "thinking_budget_tokens": config.thinking_budget_tokens, + "reasoning_effort": config.reasoning_effort, + "max_output_tokens": config.max_output_tokens, + "temperature": "omitted", + "summaries": "disabled", + } + + def write_json(path: Path, value: object) -> None: """Save one readable private evidence artifact.""" path.write_text( @@ -58,13 +76,16 @@ def positive_cost(value: str) -> Decimal: return amount -def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: +def execute( + run: Path, manifest: dict[str, Any], api_key: str, + request_config: LLMRequestConfig, +) -> bool: """Make each paid request once, saving reported usage and responses as received.""" artifacts = run / "artifacts" schedule = PriceSchedule.load(artifacts / "pricing.json") client = LLMRequestClient( api_key=api_key, - request_config=REQUEST_CONFIG, + request_config=request_config, max_retries=0, ) usage: dict[str, int] = {key: 0 for key in USAGE_KEYS} @@ -74,7 +95,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: complete = True reason = None for prompt in manifest["prompts"]: - if Decimal(schedule.estimate(MODEL, usage)) >= Decimal( + if Decimal(schedule.estimate(request_config.model, usage)) >= Decimal( manifest["max_cost_usd"] ): complete, reason = False, "cost guardrail exhausted" @@ -103,7 +124,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: name, str(result.get("response") or ""), run_id=manifest["run_id"], - model=MODEL, + model=request_config.model, page_url=manifest["source_url"], ) findings.extend(asdict(finding) for finding in normalized.findings) @@ -113,7 +134,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: complete, reason = False, str(result.get("error") or "request failed") manifest.update( usage=usage, - estimated_cost_usd=schedule.estimate(MODEL, usage), + estimated_cost_usd=schedule.estimate(request_config.model, usage), complete=False, incomplete_reason=reason, format_failures=format_failures, @@ -126,7 +147,7 @@ def execute(run: Path, manifest: dict[str, Any], api_key: str) -> bool: manifest.update( complete=complete, incomplete_reason=reason, - estimated_cost_usd=schedule.estimate(MODEL, usage), + estimated_cost_usd=schedule.estimate(request_config.model, usage), ) write_json(run / "run.json", manifest) return complete diff --git a/vision_aid/evaluation/pricing.json b/vision_aid/evaluation/pricing.json index 6e7ed53..c8fed30 100644 --- a/vision_aid/evaluation/pricing.json +++ b/vision_aid/evaluation/pricing.json @@ -5,9 +5,21 @@ "output_per_million_usd": "5", "cached_input_per_million_usd": "0.10", "cache_creation_input_per_million_usd": "1.25" + }, + "claude-opus-5-5": { + "input_per_million_usd": "4", + "output_per_million_usd": "20", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "5" + }, + "claude-sonnet-5-5": { + "input_per_million_usd": "2", + "output_per_million_usd": "10", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "2.50" } }, - "published_at": "2026-09-22", + "captured_at": "2026-10-01", "source": "https://platform.claude.com/docs/en/about-claude/pricing", - "version": "anthropic-public-2026-09-22" + "version": "anthropic-public-2026-10-01" } diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index 4083125..1985000 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -40,6 +40,13 @@ def version(self) -> str: """Return the curator-assigned price-schedule version.""" return str(self.data["version"]) + @property + def price_description(self) -> str: + """Distinguish captured public rates from older publication metadata.""" + if "captured_at" in self.data: + return f"published prices captured on {self.data['captured_at']}" + return f"published prices dated {self.data['published_at']}" + def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: """Price reported categories without rounding away tie precision.""" try: diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 08f2e50..7a4f92e 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -145,7 +145,7 @@ def write_report( llm: list[dict[str, Any]], programmatic: list[dict[str, Any]], reviewer: str, - pricing_date: str, + pricing_description: str, ) -> None: """Explain the review's catches, misses, and scoring in source order.""" if not manifest["complete"]: @@ -188,7 +188,7 @@ def write_report( lines.extend([ _rate("Overall", len(covered), len(rows)), "", - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from {', '.join(tokens)} multiplied by the published prices dated {_md(pricing_date)}. The actual bill may differ slightly.", + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from {', '.join(tokens)} multiplied by the {_md(pricing_description)}. The actual bill may differ slightly.", "", ]) if manifest["format_failures"]: From 896ccebea88188fa9d7329f9e5a71ab494eec0ed Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 11:39:49 -0400 Subject: [PATCH 056/126] fix(evaluation): explain presets and cost limit in CLI help --- tests/test_evaluation_cli.py | 6 +++++- vision_aid/evaluation/cli.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index a59498a..f2a7501 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -266,7 +266,11 @@ def test_preview_preserves_human_findings_and_never_calls_provider( with pytest.raises(SystemExit) as help_exit: main(["run", "--help"]) assert help_exit.value.code == 0 - assert "Human audit" in capsys.readouterr().out + help_text = " ".join(capsys.readouterr().out.split()) + assert "Human audit" in help_text + assert "16,000" in help_text and "24,192" in help_text + assert "adaptive thinking with high effort" in help_text + assert "a request underway can exceed" in help_text with pytest.raises(SystemExit): main([*args(human_audit, html), "--workbook", str(human_audit)]) with pytest.raises(SystemExit): diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 25a1a51..847b5d1 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -39,7 +39,12 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="visionaid-evaluate") commands = parser.add_subparsers(dest="command", required=True) run = commands.add_parser( - "run", help="preview or execute an Evaluation run with one selected model" + "run", help="preview or execute an Evaluation run with one selected model", + description=( + "Haiku uses a 16,000-token thinking budget; Opus and Sonnet use " + "adaptive thinking with high effort. All presets omit temperature, " + "disable summaries, and cap output at 24,192 tokens including thinking." + ), ) run.add_argument( "--model", choices=MODEL_PRESETS, default=DEFAULT_MODEL, @@ -51,7 +56,8 @@ def build_parser() -> argparse.ArgumentParser: run.add_argument("--html", type=Path, required=True, help="local Benchmark HTML") run.add_argument("--source-url", required=True, help="URL represented by the HTML") run.add_argument( - "--max-cost-usd", required=True, help="positive between-request limit" + "--max-cost-usd", required=True, + help="positive between-request limit; a request underway can exceed it", ) run.add_argument("--live", action="store_true", help="enable billable execution") run.add_argument( From 5994cda43dc5ba26f41d06ff5f69c71cbb3ebb5e Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 11:41:06 -0400 Subject: [PATCH 057/126] docs(evaluation): resolve model selection spec after review --- .scratch/evaluation-model-selection/review.md | 27 ++++++++++++++++++ .scratch/evaluation-model-selection/spec.md | 28 +++++++++++++------ 2 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 .scratch/evaluation-model-selection/review.md diff --git a/.scratch/evaluation-model-selection/review.md b/.scratch/evaluation-model-selection/review.md new file mode 100644 index 0000000..b9f6ef1 --- /dev/null +++ b/.scratch/evaluation-model-selection/review.md @@ -0,0 +1,27 @@ +# Evaluation model selection review + +Fixed starting commit: `49b50c522b70f788933271dbac4388451055ec58`. + +Implementation commit: `bd1d9f68`. Review-fix commit: `896ccebe`. + +The Standards and Spec axes ran in independent parallel reviewers against `git diff 49b50c522b70f788933271dbac4388451055ec58...HEAD`, using [the agreed spec](spec.md). + +## Standards + +No findings. + +Documented standards: the changes use the domain vocabulary in `CONTEXT.md`, preserve both ADRs' review/scoring boundaries, and place the feature spec under the required `.scratch//spec.md` layout. Closure remains a separate post-review step. + +Baseline smells: no actionable findings in the changed hunks. The three presets share their configuration cleanly, selected settings flow through execution explicitly, and pricing stays within the existing `PriceSchedule` abstraction. + +## Spec + +One partial requirement in the initial implementation: + +- **[P3] Expand evaluator CLI help.** The help listed model choices and the default but omitted preset settings and possible cost-limit overshoot. The spec requires evaluator help and README instructions to describe selection, the default, settings, and the between-request cost limit, including possible overshoot. README and execution output satisfied this; `run --help` needed expansion. + +No other concrete Spec findings or scope creep were identified. Model selection, streaming request shapes, saved evidence, exact pricing, lifecycle behavior, and historical report compatibility match the agreed spec. + +The reviewer verified `896ccebe`: CLI help now describes all preset settings and explains guardrail overshoot, with assertions covering both. The finding is resolved. No remaining Spec findings. + +Standards: 0 findings. Spec: 1 P3 finding, resolved. No outstanding findings on either axis. diff --git a/.scratch/evaluation-model-selection/spec.md b/.scratch/evaluation-model-selection/spec.md index 5956d3d..d3f7d13 100644 --- a/.scratch/evaluation-model-selection/spec.md +++ b/.scratch/evaluation-model-selection/spec.md @@ -1,5 +1,7 @@ # Select a model for an Evaluation run +Status: resolved + Design decisions settled and implementation authorized on October 1, 2026. ## Settled decisions @@ -73,15 +75,15 @@ This feature uses standard pricing and does not add fast mode, batch execution, ## Acceptance criteria -- [ ] Omitted `--model` selects the existing Haiku preset; each of the three exact IDs is accepted in preview and live paths. -- [ ] An unsupported ID is rejected before creating a run or making provider requests, and the error lists supported choices. -- [ ] Every selected model uses its prescribed request settings and appears consistently in run metadata, request evidence, LLM findings, cost calculations, and the final report. -- [ ] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `high` effort for both new models, omitted temperature, streaming, and final-text extraction. -- [ ] Cost tests verify each model's token categories, including Opus cache-read pricing, using saved versioned prices. -- [ ] Preview, declined approval, failure, and cost exhaustion retain existing lifecycle behavior for selected models. -- [ ] Previously completed Haiku evidence still produces a report. -- [ ] Shared-client regression checks confirm existing production caller defaults remain intact. -- [ ] CLI help and README show model selection and explain that the guardrail can be exceeded by a request underway. +- [x] Omitted `--model` selects the existing Haiku preset; each of the three exact IDs is accepted in preview and live paths. +- [x] An unsupported ID is rejected before creating a run or making provider requests, and the error lists supported choices. +- [x] Every selected model uses its prescribed request settings and appears consistently in run metadata, request evidence, LLM findings, cost calculations, and the final report. +- [x] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `high` effort for both new models, omitted temperature, streaming, and final-text extraction. +- [x] Cost tests verify each model's token categories, including Opus cache-read pricing, using saved versioned prices. +- [x] Preview, declined approval, failure, and cost exhaustion retain existing lifecycle behavior for selected models. +- [x] Previously completed Haiku evidence still produces a report. +- [x] Shared-client regression checks confirm existing production caller defaults remain intact. +- [x] CLI help and README show model selection and explain that the guardrail can be exceeded by a request underway. Verification uses mocked providers and local fixtures; this feature does not require a paid verification run. @@ -106,3 +108,11 @@ None. - [Anthropic model IDs and versioning](https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions), checked October 1, 2026. Opus defaults to medium effort and Sonnet to high. Explicit high effort for both is the agreed evaluation preset. Equal effort labels and output caps do not imply equal computation across models. + +## Comments + +Completed October 1, 2026, in implementation commit `bd1d9f68` and review-fix commit `896ccebe` on `feat/model-evaluation`. + +Verification: `uv run pytest -q` passed all 69 tests after the review fix. `uvx ty check processing_scripts/llm_client/audit.py vision_aid/evaluation --output-format concise` passed. `git diff --check` passed. Provider tests used synthetic responses; no paid verification requests were made. + +The independent Standards review had no findings. The Spec review found one missing CLI-help requirement, fixed in `896ccebe` and confirmed resolved by the reviewer. See [the review record](review.md). From f9e2186591dbfb62bc18c3c0f2542ff5a15112a8 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 12:02:36 -0400 Subject: [PATCH 058/126] fix(evaluation): use medium effort for Opus and Sonnet --- .scratch/evaluation-model-selection/spec.md | 12 ++++++------ README.md | 4 ++-- tests/test_audit_request.py | 4 ++-- tests/test_evaluation_cli.py | 14 +++++++------- vision_aid/evaluation/cli.py | 2 +- vision_aid/evaluation/llm_audit.py | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.scratch/evaluation-model-selection/spec.md b/.scratch/evaluation-model-selection/spec.md index d3f7d13..8240797 100644 --- a/.scratch/evaluation-model-selection/spec.md +++ b/.scratch/evaluation-model-selection/spec.md @@ -11,7 +11,7 @@ Design decisions settled and implementation authorized on October 1, 2026. - Each Evaluation run holds its selected model fixed throughout. A command that evaluates several models is outside this feature. - Accept only the three canonical API IDs below. Reject unsupported IDs, including convenience aliases, with an error listing the supported choices. - Use fixed request presets with no operator overrides for thinking, effort, or output token limits. -- Preserve Haiku's existing thinking budget. Use adaptive thinking with explicit `high` effort for both Opus and Sonnet. +- Preserve Haiku's existing thinking budget. Use adaptive thinking with explicit `medium` effort for both Opus and Sonnet. - Keep the existing 24,192 output-token cap for all three models. This cap includes thinking and final text; adaptive thinking does not reserve a fixed number of final-text tokens. - Preserve the positive cost guardrail checked between requests. A request underway can take the final Estimated run cost above the limit. - Add versioned public prices for the new models, and show the selected model's settings and rates before Live-run approval. @@ -23,8 +23,8 @@ This changes the Haiku-only scope in [the POC reduction spec](../poc-reduction/s | Model | Accepted `--model` value | Thinking | Effort | Output-token cap | | --- | --- | --- | --- | --- | | Haiku 4.5, default | `claude-haiku-4-5-20251001` | Enabled, 16,000-token budget | Omitted | 24,192 | -| Opus 5.5 | `claude-opus-5-5` | Adaptive, no manual budget | `high` | 24,192 | -| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive, no manual budget | `high` | 24,192 | +| Opus 5.5 | `claude-opus-5-5` | Adaptive, no manual budget | `medium` | 24,192 | +| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive, no manual budget | `medium` | 24,192 | All presets omit temperature, disable summaries, and use standard Anthropic Messages requests. These exact IDs identify fixed model snapshots. The dateless Opus and Sonnet IDs do not require invented date suffixes. @@ -67,7 +67,7 @@ This feature uses standard pricing and does not add fast mode, batch execution, ## Implementation constraints - Keep model selection and the three-model restriction in the evaluator. The shared client remains usable by existing production callers with their current defaults. -- Add explicit support for adaptive thinking and Anthropic effort in the shared request configuration. For the new models, send adaptive thinking and `output_config` with `effort` set to `high`; omit `budget_tokens`. +- Add explicit support for adaptive thinking and Anthropic effort in the shared request configuration. For the new models, send adaptive thinking and `output_config` with `effort` set to `medium`; omit `budget_tokens`. - Stream all three evaluation presets. The Anthropic Python SDK requires streaming above 21,333 output tokens, and all presets retain a 24,192 cap. Streaming must not depend only on the presence of a manual thinking budget. - Continue extracting final text separately from thinking content and preserving reported usage and failures in raw evidence. - No model-discovery or dynamic-pricing subsystem is required. Adding another model later requires an explicit preset and versioned prices. @@ -78,7 +78,7 @@ This feature uses standard pricing and does not add fast mode, batch execution, - [x] Omitted `--model` selects the existing Haiku preset; each of the three exact IDs is accepted in preview and live paths. - [x] An unsupported ID is rejected before creating a run or making provider requests, and the error lists supported choices. - [x] Every selected model uses its prescribed request settings and appears consistently in run metadata, request evidence, LLM findings, cost calculations, and the final report. -- [x] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `high` effort for both new models, omitted temperature, streaming, and final-text extraction. +- [x] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `medium` effort for both new models, omitted temperature, streaming, and final-text extraction. - [x] Cost tests verify each model's token categories, including Opus cache-read pricing, using saved versioned prices. - [x] Preview, declined approval, failure, and cost exhaustion retain existing lifecycle behavior for selected models. - [x] Previously completed Haiku evidence still produces a report. @@ -107,7 +107,7 @@ None. - [Anthropic effort configuration](https://platform.claude.com/docs/en/build-with-claude/effort), checked October 1, 2026. - [Anthropic model IDs and versioning](https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions), checked October 1, 2026. -Opus defaults to medium effort and Sonnet to high. Explicit high effort for both is the agreed evaluation preset. Equal effort labels and output caps do not imply equal computation across models. +Opus defaults to medium effort and Sonnet to high. Explicit medium effort for both is the agreed evaluation preset. Equal effort labels and output caps do not imply equal computation across models. ## Comments diff --git a/README.md b/README.md index 17746b3..8b10278 100644 --- a/README.md +++ b/README.md @@ -248,8 +248,8 @@ accepts only these exact IDs: | Model | `--model` value | Thinking | Effort | | --- | --- | --- | --- | | Haiku 4.5, default | `claude-haiku-4-5-20251001` | 16,000-token budget | Omitted | -| Opus 5.5 | `claude-opus-5-5` | Adaptive | High | -| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | High | +| Opus 5.5 | `claude-opus-5-5` | Adaptive | Medium | +| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | Medium | For example, add `--model claude-opus-5-5` to the run command below to evaluate Opus. Friendly names and aliases are rejected. Thinking settings are fixed diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index dc99246..640dbe3 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -264,8 +264,8 @@ def test_pipeline_persists_explicit_request_configuration(self): ) @pytest.mark.parametrize("model,thinking,effort", [ ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None), - ("claude-opus-5-5", {"type": "adaptive"}, "high"), - ("claude-sonnet-5-5", {"type": "adaptive"}, "high"), + ("claude-opus-5-5", {"type": "adaptive"}, "medium"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "medium"), ]) def test_thinking_stream_exposes_only_final_text(parts, stop, model, thinking, effort): """Only final text and inclusive billed output leave a thinking stream.""" diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py index f2a7501..f198287 100644 --- a/tests/test_evaluation_cli.py +++ b/tests/test_evaluation_cli.py @@ -92,8 +92,8 @@ def write_review(path: Path, rows: list[dict[str, str]]) -> None: @pytest.mark.parametrize("model,thinking,effort,input_rate,output_rate", [ ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "1", "5"), - ("claude-opus-5-5", {"type": "adaptive"}, "high", "4", "20"), - ("claude-sonnet-5-5", {"type": "adaptive"}, "high", "2", "10"), + ("claude-opus-5-5", {"type": "adaptive"}, "medium", "4", "20"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "medium", "2", "10"), ]) def test_selected_model_preview_records_preset_without_provider_access( tmp_path, monkeypatch, capsys, model, thinking, effort, input_rate, output_rate, @@ -123,7 +123,7 @@ def test_selected_model_preview_records_preset_without_provider_access( output = capsys.readouterr().out assert model in output and "24192" in output assert f"input ${input_rate}; output ${output_rate}" in output - assert "16000" in output if effort is None else "adaptive; effort: high" in output + assert "16000" in output if effort is None else "adaptive; effort: medium" in output assert "request underway can exceed" in output @@ -147,8 +147,8 @@ def test_unsupported_model_is_rejected_without_run_or_provider_access( @pytest.mark.parametrize("model,thinking,effort,cost", [ ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "0.006525"), - ("claude-opus-5-5", {"type": "adaptive"}, "high", "0.0258"), - ("claude-sonnet-5-5", {"type": "adaptive"}, "high", "0.01305"), + ("claude-opus-5-5", {"type": "adaptive"}, "medium", "0.0258"), + ("claude-sonnet-5-5", {"type": "adaptive"}, "medium", "0.01305"), ]) def test_selected_model_executes_and_reports_with_saved_prices( tmp_path, monkeypatch, model, thinking, effort, cost, @@ -167,7 +167,7 @@ def create(**kwargs): assert kwargs["max_tokens"] == 24192 and kwargs["stream"] is True assert "temperature" not in kwargs if effort: - assert kwargs["output_config"] == {"effort": "high"} + assert kwargs["output_config"] == {"effort": "medium"} else: assert "output_config" not in kwargs return nullcontext(iter([ @@ -269,7 +269,7 @@ def test_preview_preserves_human_findings_and_never_calls_provider( help_text = " ".join(capsys.readouterr().out.split()) assert "Human audit" in help_text assert "16,000" in help_text and "24,192" in help_text - assert "adaptive thinking with high effort" in help_text + assert "adaptive thinking with medium effort" in help_text assert "a request underway can exceed" in help_text with pytest.raises(SystemExit): main([*args(human_audit, html), "--workbook", str(human_audit)]) diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 847b5d1..fc3172c 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -42,7 +42,7 @@ def build_parser() -> argparse.ArgumentParser: "run", help="preview or execute an Evaluation run with one selected model", description=( "Haiku uses a 16,000-token thinking budget; Opus and Sonnet use " - "adaptive thinking with high effort. All presets omit temperature, " + "adaptive thinking with medium effort. All presets omit temperature, " "disable summaries, and cap output at 24,192 tokens including thinking." ), ) diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index ee091f0..5c12f08 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -27,7 +27,7 @@ model: LLMRequestConfig( model=model, adaptive_thinking=True, - reasoning_effort="high", + reasoning_effort="medium", max_output_tokens=24192, temperature=None, ) From d61ed556b2322e5fb708dc3450f1dfb9c0477596 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 12:03:33 -0400 Subject: [PATCH 059/126] docs(evaluation): record medium effort verification --- .scratch/evaluation-model-selection/review.md | 14 ++++++++++++++ .scratch/evaluation-model-selection/spec.md | 2 ++ 2 files changed, 16 insertions(+) diff --git a/.scratch/evaluation-model-selection/review.md b/.scratch/evaluation-model-selection/review.md index b9f6ef1..3061d35 100644 --- a/.scratch/evaluation-model-selection/review.md +++ b/.scratch/evaluation-model-selection/review.md @@ -25,3 +25,17 @@ No other concrete Spec findings or scope creep were identified. Model selection, The reviewer verified `896ccebe`: CLI help now describes all preset settings and explains guardrail overshoot, with assertions covering both. The finding is resolved. No remaining Spec findings. Standards: 0 findings. Spec: 1 P3 finding, resolved. No outstanding findings on either axis. + +## Medium effort follow-up + +The operator requested medium effort for both Opus and Sonnet. Reviewed commit `f9e21865` against starting commit `5994cda43dc5ba26f41d06ff5f69c71cbb3ebb5e`. + +### Standards + +No findings or actionable baseline smells. The shared preset, CLI help, README, spec, and provider assertions consistently use medium effort. Domain vocabulary and ADR boundaries remain intact. + +### Spec + +No findings. Presets, request assertions, metadata expectations, CLI help, README, and the updated spec consistently specify medium effort for both models. No missing requirements, incorrect behavior, or scope creep identified. + +Standards: 0 findings. Spec: 0 findings. All 69 tests and scoped typechecks pass. diff --git a/.scratch/evaluation-model-selection/spec.md b/.scratch/evaluation-model-selection/spec.md index 8240797..ee0410b 100644 --- a/.scratch/evaluation-model-selection/spec.md +++ b/.scratch/evaluation-model-selection/spec.md @@ -116,3 +116,5 @@ Completed October 1, 2026, in implementation commit `bd1d9f68` and review-fix co Verification: `uv run pytest -q` passed all 69 tests after the review fix. `uvx ty check processing_scripts/llm_client/audit.py vision_aid/evaluation --output-format concise` passed. `git diff --check` passed. Provider tests used synthetic responses; no paid verification requests were made. The independent Standards review had no findings. The Spec review found one missing CLI-help requirement, fixed in `896ccebe` and confirmed resolved by the reviewer. See [the review record](review.md). + +On October 1, 2026, the operator requested medium effort for both Opus and Sonnet. Commit `f9e21865` updates both presets, CLI help, README, this spec, and request/evidence tests. All 69 tests and scoped typechecks passed. Independent Standards and Spec reviews found no issues in this follow-up. From 605a6941b81b294e5d8c18525fb4b0e1b1c703e0 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 17:07:09 -0400 Subject: [PATCH 060/126] tests: consolidate evaluator suite and make scratch tracker local Replace the detailed evaluator CLI and normalization suites with a compact CLI integration suite that activates all 18 non-summary prompt categories. Restrict default pytest discovery to tests/, ignore and untrack .scratch/ and the optional .local-tests/ folder, and document both test commands and the local-only issue tracker. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 4 + .scratch/evaluation-model-selection/review.md | 41 - .scratch/evaluation-model-selection/spec.md | 120 --- ...ename-to-human-audit-and-llm-vocabulary.md | 33 - .../issues/02-plain-language-report.md | 75 -- .../issues/03-live-verification-run.md | 30 - .scratch/plain-language-report/spec.md | 143 ---- .scratch/poc-reduction/spec.md | 122 --- .scratch/rename-audit-artifacts/spec.md | 73 -- .scratch/run-directory-layout/spec.md | 151 ---- README.md | 9 +- docs/agents/issue-tracker.md | 8 +- pyproject.toml | 1 + tests/test_evaluation.py | 338 ++++++++ tests/test_evaluation_cli.py | 762 ------------------ tests/test_normalization.py | 133 --- 16 files changed, 356 insertions(+), 1687 deletions(-) delete mode 100644 .scratch/evaluation-model-selection/review.md delete mode 100644 .scratch/evaluation-model-selection/spec.md delete mode 100644 .scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md delete mode 100644 .scratch/plain-language-report/issues/02-plain-language-report.md delete mode 100644 .scratch/plain-language-report/issues/03-live-verification-run.md delete mode 100644 .scratch/plain-language-report/spec.md delete mode 100644 .scratch/poc-reduction/spec.md delete mode 100644 .scratch/rename-audit-artifacts/spec.md delete mode 100644 .scratch/run-directory-layout/spec.md create mode 100644 tests/test_evaluation.py delete mode 100644 tests/test_evaluation_cli.py delete mode 100644 tests/test_normalization.py diff --git a/.gitignore b/.gitignore index 3784651..52d8105 100644 --- a/.gitignore +++ b/.gitignore @@ -225,3 +225,7 @@ web/prisma/generated/ # Stray git worktrees — never belong in the index .claude/worktrees/ + +# Local-only issue tracker and detailed evaluator tests +/.scratch/ +/.local-tests/ diff --git a/.scratch/evaluation-model-selection/review.md b/.scratch/evaluation-model-selection/review.md deleted file mode 100644 index 3061d35..0000000 --- a/.scratch/evaluation-model-selection/review.md +++ /dev/null @@ -1,41 +0,0 @@ -# Evaluation model selection review - -Fixed starting commit: `49b50c522b70f788933271dbac4388451055ec58`. - -Implementation commit: `bd1d9f68`. Review-fix commit: `896ccebe`. - -The Standards and Spec axes ran in independent parallel reviewers against `git diff 49b50c522b70f788933271dbac4388451055ec58...HEAD`, using [the agreed spec](spec.md). - -## Standards - -No findings. - -Documented standards: the changes use the domain vocabulary in `CONTEXT.md`, preserve both ADRs' review/scoring boundaries, and place the feature spec under the required `.scratch//spec.md` layout. Closure remains a separate post-review step. - -Baseline smells: no actionable findings in the changed hunks. The three presets share their configuration cleanly, selected settings flow through execution explicitly, and pricing stays within the existing `PriceSchedule` abstraction. - -## Spec - -One partial requirement in the initial implementation: - -- **[P3] Expand evaluator CLI help.** The help listed model choices and the default but omitted preset settings and possible cost-limit overshoot. The spec requires evaluator help and README instructions to describe selection, the default, settings, and the between-request cost limit, including possible overshoot. README and execution output satisfied this; `run --help` needed expansion. - -No other concrete Spec findings or scope creep were identified. Model selection, streaming request shapes, saved evidence, exact pricing, lifecycle behavior, and historical report compatibility match the agreed spec. - -The reviewer verified `896ccebe`: CLI help now describes all preset settings and explains guardrail overshoot, with assertions covering both. The finding is resolved. No remaining Spec findings. - -Standards: 0 findings. Spec: 1 P3 finding, resolved. No outstanding findings on either axis. - -## Medium effort follow-up - -The operator requested medium effort for both Opus and Sonnet. Reviewed commit `f9e21865` against starting commit `5994cda43dc5ba26f41d06ff5f69c71cbb3ebb5e`. - -### Standards - -No findings or actionable baseline smells. The shared preset, CLI help, README, spec, and provider assertions consistently use medium effort. Domain vocabulary and ADR boundaries remain intact. - -### Spec - -No findings. Presets, request assertions, metadata expectations, CLI help, README, and the updated spec consistently specify medium effort for both models. No missing requirements, incorrect behavior, or scope creep identified. - -Standards: 0 findings. Spec: 0 findings. All 69 tests and scoped typechecks pass. diff --git a/.scratch/evaluation-model-selection/spec.md b/.scratch/evaluation-model-selection/spec.md deleted file mode 100644 index ee0410b..0000000 --- a/.scratch/evaluation-model-selection/spec.md +++ /dev/null @@ -1,120 +0,0 @@ -# Select a model for an Evaluation run - -Status: resolved - -Design decisions settled and implementation authorized on October 1, 2026. - -## Settled decisions - -- Support a hardcoded set of three models through Anthropic's direct API: Haiku 4.5, Opus 5.5, and Sonnet 5.5. -- Select one model per Evaluation run through `--model`; retain Haiku 4.5 as the default. -- Each Evaluation run holds its selected model fixed throughout. A command that evaluates several models is outside this feature. -- Accept only the three canonical API IDs below. Reject unsupported IDs, including convenience aliases, with an error listing the supported choices. -- Use fixed request presets with no operator overrides for thinking, effort, or output token limits. -- Preserve Haiku's existing thinking budget. Use adaptive thinking with explicit `medium` effort for both Opus and Sonnet. -- Keep the existing 24,192 output-token cap for all three models. This cap includes thinking and final text; adaptive thinking does not reserve a fixed number of final-text tokens. -- Preserve the positive cost guardrail checked between requests. A request underway can take the final Estimated run cost above the limit. -- Add versioned public prices for the new models, and show the selected model's settings and rates before Live-run approval. - -This changes the Haiku-only scope in [the POC reduction spec](../poc-reduction/spec.md). Human review and scoring continue to follow [ADR 0001](../../docs/adr/0001-separate-review-from-scoring.md) and [ADR 0002](../../docs/adr/0002-rank-models-by-llm-detection-rate.md). - -## Model presets - -| Model | Accepted `--model` value | Thinking | Effort | Output-token cap | -| --- | --- | --- | --- | --- | -| Haiku 4.5, default | `claude-haiku-4-5-20251001` | Enabled, 16,000-token budget | Omitted | 24,192 | -| Opus 5.5 | `claude-opus-5-5` | Adaptive, no manual budget | `medium` | 24,192 | -| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive, no manual budget | `medium` | 24,192 | - -All presets omit temperature, disable summaries, and use standard Anthropic Messages requests. These exact IDs identify fixed model snapshots. The dateless Opus and Sonnet IDs do not require invented date suffixes. - -Example selection: - -```bash -uv run visionaid-evaluate run \ - --html .model-evaluation/pristine-home.html \ - --source-url https://pristineai.com/ \ - --model claude-opus-5-5 \ - --max-cost-usd 1.00 -``` - -This example prepares a preview. The existing `--live` and Live-run approval flow enables paid execution. - -## Run behavior and evidence - -- Resolve the selected preset once and use it consistently for prompt preparation, paid execution, normalization, cost estimates, and saved run metadata. Execution must not silently fall back to global Haiku settings. -- Record the canonical model ID, provider and endpoint, thinking mode, manual budget when applicable, effort when applicable, output cap, omitted temperature, and disabled summaries in `run.json`. Retain provider-returned model identity in raw response evidence. -- Display the selected preset, applicable token rates, versioned price-schedule identity, prompt count, and positive cost guardrail before asking for Live-run approval. -- Preserve key-free previews, no requests after declined approval, explicit authorization for paid execution, sequential requests, disabled retries, saved partial evidence, and stopping on request failure or accumulated cost exhaustion. -- A model that is unavailable to the operator's account fails with retained evidence under the existing request-failure behavior. Do not substitute another model. -- Continue to generate reports from saved run evidence, including previously completed Haiku runs whose metadata predates model selection. Human review and detection-rate scoring do not change. -- Preserve the current handling of malformed responses and output-limit stop reasons; changing completion semantics is outside this feature. - -## Versioned pricing - -Standard public rates checked October 1, 2026, in USD per million tokens: - -| Model | Input | Output | Cache read | 5-minute cache write | -| --- | ---: | ---: | ---: | ---: | -| Haiku 4.5 | 1 | 5 | 0.10 | 1.25 | -| Opus 5.5 | 4 | 20 | 0.20 | 5 | -| Sonnet 5.5 | 2 | 10 | 0.20 | 2.50 | - -Use exact rates from the recorded schedule, including Opus's cache-read rate rather than assuming a common multiplier. Keep the schedule copy and its content identity with each run. Existing run evidence retains its original schedule. Record the rate source and capture date accurately; a capture date is not necessarily the publisher's publication date. - -This feature uses standard pricing and does not add fast mode, batch execution, residency options, or new caching behavior. - -## Implementation constraints - -- Keep model selection and the three-model restriction in the evaluator. The shared client remains usable by existing production callers with their current defaults. -- Add explicit support for adaptive thinking and Anthropic effort in the shared request configuration. For the new models, send adaptive thinking and `output_config` with `effort` set to `medium`; omit `budget_tokens`. -- Stream all three evaluation presets. The Anthropic Python SDK requires streaming above 21,333 output tokens, and all presets retain a 24,192 cap. Streaming must not depend only on the presence of a manual thinking budget. -- Continue extracting final text separately from thinking content and preserving reported usage and failures in raw evidence. -- No model-discovery or dynamic-pricing subsystem is required. Adding another model later requires an explicit preset and versioned prices. -- Update evaluator help and README instructions to describe selection, the default, settings, and the existing between-request cost limit. - -## Acceptance criteria - -- [x] Omitted `--model` selects the existing Haiku preset; each of the three exact IDs is accepted in preview and live paths. -- [x] An unsupported ID is rejected before creating a run or making provider requests, and the error lists supported choices. -- [x] Every selected model uses its prescribed request settings and appears consistently in run metadata, request evidence, LLM findings, cost calculations, and the final report. -- [x] Mock-provider tests verify the exact Haiku and adaptive request shapes, explicit `medium` effort for both new models, omitted temperature, streaming, and final-text extraction. -- [x] Cost tests verify each model's token categories, including Opus cache-read pricing, using saved versioned prices. -- [x] Preview, declined approval, failure, and cost exhaustion retain existing lifecycle behavior for selected models. -- [x] Previously completed Haiku evidence still produces a report. -- [x] Shared-client regression checks confirm existing production caller defaults remain intact. -- [x] CLI help and README show model selection and explain that the guardrail can be exceeded by a request underway. - -Verification uses mocked providers and local fixtures; this feature does not require a paid verification run. - -## Remaining design decisions - -None. - -## Starting code facts - -- `vision_aid/evaluation/llm_audit.py` pins Haiku, its request settings, and the model used for normalization and cost estimation. -- `vision_aid/evaluation/cli.py` has no model selection option and uses those fixed settings for preparation, run metadata, and Live-run approval. -- The versioned price schedule contains only the pinned Haiku ID. Cost estimation requires an exact model match. -- The shared request client supports manual extended thinking but has no adaptive thinking or Anthropic effort settings. -- Reports already read the model from saved run metadata. - -## Sources - -- [Anthropic model overview](https://platform.claude.com/docs/en/models/overview), checked October 1, 2026. -- [Anthropic pricing](https://platform.claude.com/docs/en/about-claude/pricing), checked October 1, 2026. -- [Anthropic thinking configuration and streaming constraints](https://platform.claude.com/docs/en/build-with-claude/thinking), checked October 1, 2026. -- [Anthropic effort configuration](https://platform.claude.com/docs/en/build-with-claude/effort), checked October 1, 2026. -- [Anthropic model IDs and versioning](https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions), checked October 1, 2026. - -Opus defaults to medium effort and Sonnet to high. Explicit medium effort for both is the agreed evaluation preset. Equal effort labels and output caps do not imply equal computation across models. - -## Comments - -Completed October 1, 2026, in implementation commit `bd1d9f68` and review-fix commit `896ccebe` on `feat/model-evaluation`. - -Verification: `uv run pytest -q` passed all 69 tests after the review fix. `uvx ty check processing_scripts/llm_client/audit.py vision_aid/evaluation --output-format concise` passed. `git diff --check` passed. Provider tests used synthetic responses; no paid verification requests were made. - -The independent Standards review had no findings. The Spec review found one missing CLI-help requirement, fixed in `896ccebe` and confirmed resolved by the reviewer. See [the review record](review.md). - -On October 1, 2026, the operator requested medium effort for both Opus and Sonnet. Commit `f9e21865` updates both presets, CLI help, README, this spec, and request/evidence tests. All 69 tests and scoped typechecks passed. Independent Standards and Spec reviews found no issues in this follow-up. diff --git a/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md b/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md deleted file mode 100644 index e43005f..0000000 --- a/.scratch/plain-language-report/issues/01-rename-to-human-audit-and-llm-vocabulary.md +++ /dev/null @@ -1,33 +0,0 @@ -# 01: Rename to Human audit and LLM vocabulary (clean break) - -**What to build:** An evaluator using the evaluation CLI works entirely in the glossary vocabulary (Human audit, Human finding, LLM finding, Programmatic finding). A new Evaluation run takes the benchmark spreadsheet via `--human-audit` and writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`. Its `review.csv` has `human_finding_id` and `llm_finding_id` columns, and its IDs carry `human-`, `llm-` and `programmatic-` prefixes. The run manifest records `human_audit_filename` and `human_audit_sha256`. CLI help and error messages and the README's evaluation sections use the glossary terms. `report` reads only the new names and still produces the current report layout; the layout is rewritten in ticket 02. There is no compatibility layer and no migration of old run directories. See the Rename decisions in `../spec.md`. - -**Blocked by:** None (can start immediately). - -**Status:** resolved - -- [x] The `run` subcommand accepts `--human-audit` in place of `--workbook`, with the default-path constant renamed to match -- [x] A run writes `human-findings.json`, `raw-llm-responses.json` and `normalized-llm-findings.json`, and never writes `references.json`, `raw-audit-responses.json` or `normalized-audit-findings.json` (asserted in tests) -- [x] `review.csv` uses `human_finding_id` and `llm_finding_id` columns, and `report` validates against those columns -- [x] Human finding IDs start with `human-`, LLM finding IDs with `llm-`, and Programmatic finding IDs with `programmatic-`; the hash part of each ID is unchanged -- [x] LLM findings record `source` as `"llm"` -- [x] The import module/function and records are renamed (`human_audit`, `import_homepage_human_findings`, `HumanFinding`, `HumanAudit` with `human_findings`) -- [x] The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match -- [x] The manifest records `human_audit_filename` and `human_audit_sha256`, and `report` checks the Human audit checksum against them -- [x] CLI help and error messages say "Human audit", "Human finding" and "LLM finding" instead of "Reference workbook", "Reference defect", "Excluded Reference" or "audit finding" -- [x] The README's evaluation workflow, run-directory file table, review-column guide and troubleshooting text use the new names -- [x] Classification values, the spreadsheet's sheet and column headers, and the programmatic artifact names are unchanged -- [x] Evaluation CLI tests pass through the `main` seam with the fake LLM client, and the full test suite passes -- [x] The commit is marked breaking (`!` and a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported) - -## Comments - -### Completed 2026-09-29 - -Implementation: `401822c9a4a60ac4e196f0bef53e07432990aed5` (`refactor(evaluation)!: adopt Human audit and LLM vocabulary`). - -Verification: `uv run pytest -q tests/test_evaluation_cli.py` passed all 8 cases; `uv run pytest -q` passed all 37 tests. `uv run python -m compileall -q vision_aid/evaluation tests/test_evaluation_cli.py` and `git diff --check` passed. No dedicated typechecker is configured. Existing CLI main-seam cases verify the new Human audit flag, artifact names and legacy artifact absence, CSV columns, finding prefixes/source values, manifest provenance, legacy flag/CSV rejection, and Human audit checksum mismatch rejection using fake LLM clients. No paid requests were made. - -Review: independent Standards and Spec agents reviewed the diff from `12c17ab311ae30e62e208f08784760c5a4449c12`; both reported zero findings. Spec review confirmed unchanged ID hash construction, classification values, source sheet/headers, Programmatic artifact names, report layout, scoring, and shared pipeline. Shared request classes are imported under LLM names only within the evaluator. The implementation commit carries `!` and the required `BREAKING CHANGE:` footer. - -Size: production Python additions against main are 1,567 versus 1,562 before this ticket; all Python including tests is 2,565 versus 2,534. diff --git a/.scratch/plain-language-report/issues/02-plain-language-report.md b/.scratch/plain-language-report/issues/02-plain-language-report.md deleted file mode 100644 index 50c7fb9..0000000 --- a/.scratch/plain-language-report/issues/02-plain-language-report.md +++ /dev/null @@ -1,75 +0,0 @@ -# 02: Plain-language report - -**What to build:** A reader with no knowledge of the evaluator opens `report.md` and learns, in plain language, which Human findings the LLM and the Programmatic checks caught, what the tool said, why each catch counts, what was missed, and what could not be tested. `report` requires `--reviewer`. The report follows the layout in `../spec.md`, in this order: - -1. title; -2. How to read this report; -3. Summary; -4. Human findings table; -5. What each part of the tool caught; -6. Missed; -7. Not testable from the page file. - -It contains no IDs, CSS paths, location JSON, raw evidence JSON or raw usage JSON. Scoring semantics are unchanged. - -**Blocked by:** 01 (Rename to Human audit and LLM vocabulary) - -**Status:** resolved - -- [x] `report` fails without `--reviewer`; with it, the Summary shows "Review decisions by: " -- [x] The title is "LLM accessibility evaluation: ", and the Summary shows the model and run ID -- [x] "How to read this report" says: - - the benchmark is the Human audit; - - the tool has two parts (the LLM and programmatic checks), each defined; - - the tool saw only the saved HTML; - - a reviewer set expectations and only full coverage counts; - - the scope is the homepage only -- [x] The LLM, Programmatic and Overall detection rates each appear as a sentence with the count, the denominator explained, and a percentage; zero eligible findings is stated in words -- [x] The cost line shows dollars rounded to cents, input and output tokens, and the pricing-schedule publication date, and ends "The actual bill may differ slightly." Cached and cache-creation tokens appear only when non-zero -- [x] A sentence names the LLM checks with malformed output out of the total prompt count, and is omitted when there are none -- [x] The table has one row per Human finding in source order, with columns for the Issue Title, WCAG, expected part (LLM / Programmatic checks / Not testable), and ✓/✗ for each part; cross-catches are marked, and a footnote explains what the rates count -- [x] Each caught Human finding lists its matched findings, labelled "LLM ( check)" or "Programmatic check ", with the problem text and "Why this counts:" giving the match reason -- [x] "Missed" lists every `llm_eligible` or `programmatic` Human finding with no match, even when the classification reason and review notes are both empty; "Why it was expected to be caught" and "Why it was missed" appear only when non-empty -- [x] "Not testable from the page file" groups `unavailable_evidence` and `ambiguous` findings under one intro sentence, each with its classification reason -- [x] The report contains no Human finding or finding IDs, `source_element`, CSS paths or JSON blocks, and free text stays escaped for Markdown (inline HTML and table pipes) -- [x] Tests through the `main` seam cover each item above, including a cross-catch, a missed finding with no explanations, both not-testable classifications, and the zero-eligible wording -- [x] The README's description of `report` and its output matches the new layout, including the `--reviewer` argument - -## Comments - -- Operator correction: allow empty `classification_reason` values. A valid - classification is still required for every Human finding, and a match reason - is still required whenever a catch is claimed. The CLI test includes a missed - finding with both classification reason and review notes blank. The source - spec remains unchanged. -- Implemented in `be15c06e50b07e4033e278c4ac79cfd8248c51ba`. - `report` requires a nonblank reviewer description and emits the seven specified - sections with source-order table rows, cross-catches, matched problem text and - reasons, all misses, and the combined not-testable group. It omits detailed IDs, - source elements, locations, and raw JSON, while escaping free text. The README - documents the layout and optional classification reasons. -- Scoring clarification: the spec's explicit unchanged scoring rule takes - precedence over its contradictory exclusion prose. Not-testable findings stay - outside the LLM and Programmatic rates and remain in the Overall denominator; - the report and README say so. -- Verification: `uv run pytest -q` passed all 37 tests. The existing CLI suite - covers reviewer requirement and attribution; explanatory prose and section - order; 1/1 LLM, 1/1 Programmatic, and 2/3 Overall rates; both zero-eligible - sentences; shuffled CSV source ordering; matched source labels and reasons; - cross-catches; blank and explained misses; both not-testable classifications; - inline HTML and table-pipe escaping; omitted evidence and location fields; - nonzero cost rounded to cents, pricing date, cache counts present and absent; - and unreadable-output wording present and absent. Existing classification, - finding-ID, and required match-reason validation remain covered. - `uv run python -m compileall -q vision_aid/evaluation tests/test_evaluation_cli.py` - and `git diff --check` passed. No repository typechecker is configured; an - isolated targeted mypy check also passed for the two changed production files. -- Parallel code review against `61ac26c01fefdfc9abd5bf5ad19f981a728acfd9`: - Standards found no documented violations and one optional, low-severity - Primitive Obsession observation about two local section-title comparisons. - Kept the small renderer without added abstraction. Spec found zero findings; - the independent reviewer also reran the CLI suite, with 8 tests passing. -- Python additions against main: 1,585 production lines and 2,593 including - tests, versus the original 1,562 / 2,534 baseline. The report replaces old - verbosity and reuses existing CLI fixtures and tests. No paid calls made in - this ticket; live verification belongs to ticket 03. diff --git a/.scratch/plain-language-report/issues/03-live-verification-run.md b/.scratch/plain-language-report/issues/03-live-verification-run.md deleted file mode 100644 index 8dd7e60..0000000 --- a/.scratch/plain-language-report/issues/03-live-verification-run.md +++ /dev/null @@ -1,30 +0,0 @@ -# 03: Live verification run with an agent reviewer - -**What to build:** The operator sees the new report produced from real model output. One paid Evaluation run is made on the saved Pristine homepage snapshot. A sub-agent acting as the reviewer fills in every `review.csv` row, and the report is generated and read as an outsider would read it. The operator authorized this spend in the planning conversation; follow the spending controls in `.model-evaluation/live-run-plan.md` (sequential requests, no retries, stop on failure). - -**Blocked by:** 02 (Plain-language report) - -**Status:** resolved - -- [x] One live run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with `--max-cost-usd 1.10` completes, or its stop reason is reported without a retry -- [x] A sub-agent fills in every `review.csv` row. It classifies each Human finding against the snapshot before reading the LLM responses, then records Match decisions following the README's rules, with a reason for every classification and match -- [x] `report` succeeds with `--reviewer "Codex agent (operator-directed)"` -- [x] The generated report is read end to end, and any wording that still needs knowledge of the evaluator is reported to the operator -- [x] The run's estimated cost and detection rates are recorded under Comments - -## Comments - -2026-09-29 verification: - -- Operator corrections: attribute the available reviewer as Codex, and replace the saved homepage HTML before calculating its new checksum. The source spec and historical plan remain unchanged. -- Refreshed snapshot: 184,244 bytes, SHA-256 `a0b5f43b37ffa276b23d04da94e846c24ec2b2148ec9152ec448db7e68945c5a`. Human audit SHA-256 remains `f8e9c83d5de8155cd81c4d9c15180c6e38640a5aba7323d35040b81e39aa6c9a`. -- Report and reviewed CSV: `.model-evaluation/runs/20260929T215031-9f661935/{report.md,review.csv}`. All 16 rows reviewed by a fresh sub-agent; baseline frozen at 21:50:16 UTC before dispatch, SHA-256 `f7c1077a3e8441b1cf4e8b237145e0eb5cca550db6868a820d9b5a3d4c04ca19`. -- Independent classifications: 8 LLM-eligible, 4 programmatic, 4 unavailable, 0 ambiguous. Unlike the historical 6/6/4 plan, the complete heading-hierarchy defect and decorative text-link defect require contextual judgment. Classifications/reasons stayed frozen after dispatch; full matches require reasons and never reuse finding IDs. -- One live attempt used `--max-cost-usd 1.10`, fixed Haiku, 16,000 thinking tokens, 24,192 output cap, sequential requests and zero retries. Free counting found 7,421 input tokens; exact prepared prompts matched before paid dispatch. The private ledger reserved $1.50 for this attempt and accounted for $0.237422 of prior spending. -- All 8 requests succeeded with `end_turn`; none reached the output cap. SVG output contained valid fenced JSON followed by prose and was recorded as a format failure, without repair or retry. All raw replies and 16 LLM/29 Programmatic findings were inspected. -- Estimated cost: $0.104626 for 7,421 input and 19,441 output tokens, no cache usage, using the recorded 2026-09-22 prices reverified on 2026-09-29. Reconciled cumulative estimate $0.342048; remaining budget $4.657952. -- LLM detection 1/8 (12.5%); Programmatic detection 4/4 (100%); Overall detection 5/16 (31.25%, displayed 31.2%). Matches: Human audit findings 1, 2, 3, 7, 12; eligible misses 4, 5, 6, 8, 9, 10, 16; not testable 11, 13, 14, 15. -- `report --reviewer "Codex agent (operator-directed)"` succeeded. Reviewer, implementation agent and operator agent read it end to end; CSS selectors and internal review-phase wording were removed from live notes. Remaining reader limitations: "unreadable" means a parsing failure here; bundled findings include unrelated issues, repeat match reasons and make navigation credit unclear without knowing the no-reuse rule. Some frozen explanations retain accessibility/HTML terminology. -- Verification: `uv run pytest -q` (37 passed), `uv lock --check`, entry-point imports, and independent CSV/source/identity/rate/report checks passed. No production code or tests added. Snapshot, baseline, spending ledger and full run artifacts remain private and ignored. - -Resolved after implementation commit `e77b2c06efa9b17577605e933d5048cf6e159061`. Independent Standards and Spec reviews passed with zero findings; all acceptance criteria demonstrated by the verification recorded above. This separate closure commit records the exact implementation ID. diff --git a/.scratch/plain-language-report/spec.md b/.scratch/plain-language-report/spec.md deleted file mode 100644 index d041f2d..0000000 --- a/.scratch/plain-language-report/spec.md +++ /dev/null @@ -1,143 +0,0 @@ -# Plain-language evaluation report and Human audit naming - -Status: ready-for-agent - -## Problem Statement - -A reader of an Evaluation run's final report has to know how the evaluator works before the report makes sense. It uses undefined jargon: "Workbook-row recall", "Combined workbook coverage", "Reference defect", "Audit findings", and raw classification values such as `llm_eligible` and `unavailable_evidence`. Most of its length is spent on material the reader does not need, such as reference and finding IDs, the `source_element` field, CSS paths and location JSON, raw token-usage JSON, and a collapsible raw-evidence JSON block for every row. It never plainly says which defects the human testers reported, which of those the tool caught, which part of the tool caught each one, and which were missed. The report does not define "unavailable evidence" or "ambiguous", so a reader cannot tell why some defects were left out of the scores. - -The same jargon runs through the evaluator's code, artifact files, review CSV columns, CLI flags and README. "Workbook" names the file format of the benchmark rather than what it is: an audit performed by humans. "Audit finding" is ambiguous, because the human testers also produced an audit. - -## Solution - -The report is rewritten for a reader who knows nothing about the evaluator. It opens with a short "How to read this report" section and gives the headline numbers as plain sentences with their denominators explained. A single table then shows every Human finding and whether the LLM, the Programmatic checks, or neither caught it. For each catch the report shows what the tool said and the reviewer's reason the catch counts. Every miss is listed with background on why it was expected to be caught and why it was missed. Findings that could not be tested from the saved page are grouped under one plain heading, "Not testable from the page file", each with its reason. The estimated cost is explained as token counts multiplied by published prices. The report states who made the review decisions, so agent-made decisions are never mistaken for human adjudication. - -The evaluator adopts the glossary in `CONTEXT.md` everywhere, including code, artifact files, CSV columns, CLI flags, error messages and the README. The terms are Human audit, Human finding, LLM, LLM finding, Programmatic checks, Programmatic finding, LLM detection rate, Programmatic detection rate, Overall detection rate, and Not testable from the page file. The rename is a clean break with no migration of old run directories. - -## User Stories - -1. As a report reader with no knowledge of the evaluator, I want a short "How to read this report" section at the top, so that I understand what is being measured before I see any numbers. -2. As a report reader, I want the benchmark called the "Human audit" and each of its defects a "Human finding", so that I know the comparison is against human testers and not a spreadsheet artifact. -3. As a report reader, I want the two parts of the tool named "the LLM" and "programmatic checks", each defined in one sentence, so that I can tell AI judgment from fixed rules. -4. As a report reader, I want to be told that the tool saw only a saved copy of the page's HTML, not a live browser or screen reader, so that I understand why some human findings could not be tested. -5. As a report reader, I want to be told that a catch counts only when the tool's findings cover the whole Human finding, so that I don't read partial matches as successes. -6. As a report reader, I want to be told the evaluation covers the homepage only, so that I don't generalise the numbers to the whole site. -7. As a report reader, I want the LLM detection rate written as a sentence ("Of the 8 Human findings the LLM could be expected to detect from the page's HTML, it caught 3 (37.5%)"), so that I know what the percentage is out of. -8. As a report reader, I want the Programmatic detection rate written the same way, so that I can compare the two parts of the tool on equal terms. -9. As a report reader, I want an Overall detection rate saying how many of all Human findings were caught by either part, so that I see the tool's total reach. -10. As a report reader, I want a rate with zero eligible findings stated in words and not as a division by zero, so that the summary never shows a meaningless percentage. -11. As a report reader, I want to see the model and run ID in the summary, so that I know which configuration produced these results. -12. As a report reader, I want the estimated cost in dollars and cents with the token counts and the price-list date it came from, so that I understand it is an estimate and how it was calculated. -13. As a report reader, I want cached-token counts mentioned only when they are non-zero, so that the cost line stays short in the common case. -14. As a report reader, I want a plain sentence naming any LLM checks that returned unreadable output, so that I know some LLM misses were caused by broken responses and not poor judgment. -15. As a report reader, I want that sentence left out when every LLM check returned readable output, so that I'm not distracted by a "none" line. -16. As a report reader, I want a table with one row per Human finding showing its title, WCAG criterion, which part of the tool it was expected from, and ✓/✗ for each part, so that I see the whole outcome at a glance. -17. As a report reader, I want the table to mark every catch even when the part of the tool that was not expected to catch a finding caught it, so that I see everything the tool actually found. -18. As a report reader, I want a footnote explaining that the detection rates count only catches by the expected part of the tool, so that a cross-catch ✓ that doesn't change a rate doesn't confuse me. -19. As a report reader, I want each caught Human finding to list the matched findings, labelled by source ("LLM (link clarity check)" or "Programmatic check LAND_001") with the tool's problem text, so that I can see what the tool actually said. -20. As a report reader, I want the reviewer's reason for each catch shown as "Why this counts", so that I can judge whether the ✓ is deserved. -21. As a report reader, I want the source labels to make the rule names cited in match reasons understandable, so that a reason like "LAND_001 reports exactly this" makes sense. -22. As a report reader, I want a "Missed" section listing every counted Human finding that nothing caught, so that the tool's gaps are explicit. -23. As a report reader, I want each missed finding to show why it was expected to be caught, so that I believe it was a real miss and not an impossible task. -24. As a report reader, I want each missed finding to show why it was missed when the reviewer recorded a reason, so that I understand the failure. -25. As a report reader, I want a missed finding listed even when neither explanation was recorded, so that no miss is silently hidden. -26. As a report reader, I want findings classified as unavailable evidence or ambiguous grouped under "Not testable from the page file", with one sentence saying they are left out of every detection rate, so that I don't need to learn two internal categories. -27. As a report reader, I want each not-testable finding shown with the reviewer's specific reason, so that I know why it couldn't be tested. -28. As a report reader, I want the report to say who made the review decisions, so that I know whether a human or an agent judged the matches. -29. As a report reader, I want no IDs, CSS paths, location JSON, raw evidence JSON or raw usage JSON in the report, so that the document stays readable. -30. As a report reader, I want Human findings shown in the Human audit's original order, so that I can cross-reference the original spreadsheet. -31. As an evaluator, I want `report` to require a reviewer description, so that I can't publish a report that hides who made the decisions. -32. As an evaluator, I want the CLI flag for the benchmark spreadsheet called `--human-audit`, so that the command reads in the project's vocabulary. -33. As an evaluator, I want the imported Human findings saved as `human-findings.json`, so that the artifact name says what it holds. -34. As an evaluator, I want the review CSV columns called `human_finding_id` and `llm_finding_id`, so that I know which ID goes in which column. -35. As an evaluator, I want LLM artifacts called `raw-llm-responses.json` and `normalized-llm-findings.json`, so that they sit alongside the programmatic `raw-…`/`normalized-…` files under matching names. -36. As an evaluator, I want IDs prefixed `human-`, `llm-` and `programmatic-`, so that I can tell at a glance what kind of item an ID refers to when filling in the review CSV. -37. As an evaluator, I want CLI help and error messages to use the glossary terms ("Human finding", "LLM finding"), so that errors match the vocabulary in the CSV and README. -38. As an evaluator, I want the run manifest to record `human_audit_filename` and `human_audit_sha256`, so that its provenance fields use the same name as the concept. -39. As an evaluator, I want the README's evaluation workflow, file table, review-column guide and report description updated to the new names and layout, so that the instructions match what's on disk. -40. As an evaluator, I want the classification values (`llm_eligible`, `programmatic`, `unavailable_evidence`, `ambiguous`) unchanged in the CSV, so that the reviewer still records the finer distinction even though the report combines two of them. -41. As a maintainer, I want the rename to be a clean break with no compatibility code, so that the evaluator doesn't carry fallbacks for throwaway local output. -42. As a maintainer, I want the change marked as breaking in its commit, so that anyone holding an old run directory knows `report` will no longer read it. -43. As a maintainer, I want tests that fail if old artifact names are written again, so that the rename can't silently regress. -44. As a maintainer, I want scoring rules unchanged, so that the new report's rates equal the old metrics under new names. -45. As the operator, I want the change verified with a real paid run reviewed by an agent acting as the reviewer, so that I can judge the new report against real model output before relying on it. - -## Implementation Decisions - -- **Glossary.** `CONTEXT.md` already holds the resolved terms, and ADR 0002 has been reworded to "LLM detection rate" and renamed accordingly. The ranking decision itself is unchanged. This spec supersedes the "Out of Scope" decision in the `rename-audit-artifacts` spec that kept the name "Audit finding". -- **Report module.** The report writer in the review module is rewritten. Its inputs are the same (the manifest, reviewed rows, and LLM and Programmatic findings) plus the reviewer description and the pricing-schedule date. It emits these sections in order: - 1. title "LLM accessibility evaluation: "; - 2. How to read this report; - 3. Summary; - 4. Human findings table; - 5. What each part of the tool caught; - 6. Missed; - 7. Not testable from the page file. -- **Scoring semantics unchanged.** - - LLM detection rate: `llm_eligible` rows with at least one matched LLM finding, out of all `llm_eligible` rows. - - Programmatic detection rate: `programmatic` rows with at least one matched Programmatic finding, out of all `programmatic` rows. - - Overall detection rate: rows with any match, out of all imported Human findings, so not-testable findings stay in that denominator. - - Review validation rules are unchanged, including that not-testable rows cannot claim matches. -- **"Expected to be caught by" mapping.** `llm_eligible` shows as "LLM", `programmatic` as "Programmatic checks", and `unavailable_evidence`/`ambiguous` as "Not testable". -- **Missed definition.** A missed finding is an `llm_eligible` or `programmatic` Human finding with no matched finding of either kind. It is always listed. "Why it was expected to be caught" (the classification reason) and "Why it was missed" (the review notes) appear only when non-empty. -- **Finding source labels.** LLM findings are labelled "LLM ( check)". Programmatic findings are labelled "Programmatic check ". Both come from the finding's recorded prompt field. -- **Cost line.** - - The dollar amount is the run's estimated cost rounded to cents; the exact figure stays in the manifest. - - Input and output token counts are always shown. Cached and cache-creation counts are shown only when non-zero. - - The price date is the recorded pricing schedule's publication date, and the line ends "The actual bill may differ slightly." -- **Malformed-output sentence.** It names the LLM checks listed as format failures in the manifest, in words, out of the total prompt count, and is omitted when there are none. -- **Markdown safety.** Free text from the Human audit, review CSV and findings is still escaped for inline Markdown, including raw HTML and table pipes. -- **Reviewer attribution.** The `report` subcommand gains a required `--reviewer` text argument, printed in the summary as "Review decisions by: ". -- **Rename (clean break, no migration or compatibility layer):** - - CLI flag `--workbook` → `--human-audit`, and the default path constant renamed to match. - - Artifact `references.json` → `human-findings.json`. - - Human audit import module and function renamed to `human_audit` / `import_homepage_human_findings`. - - Records `ReferenceDefect` → `HumanFinding` and `ReferenceSet` → `HumanAudit`; its collection field becomes `human_findings`. - - Field and CSV column `reference_id` → `human_finding_id`; CSV column `audit_finding_id` → `llm_finding_id`. - - Manifest and HumanAudit fields `workbook_filename`/`workbook_sha256` → `human_audit_filename`/`human_audit_sha256`. - - Artifacts `raw-audit-responses.json` → `raw-llm-responses.json` and `normalized-audit-findings.json` → `normalized-llm-findings.json`. - - Finding `source` value `"audit"` → `"llm"`. - - The LLM execution module is renamed `llm_audit`, and its request-client names are reviewed to match. - - ID prefixes `ref-` → `human-` and `finding-` → `llm-`; `programmatic-` is unchanged. The hash portion of each ID is unchanged. - - User-facing CLI help text and error messages use the glossary terms. - - README evaluation sections are updated to the new names and report layout. -- **Unchanged:** the classification values, the Human audit's sheet name and column headers (source data), the programmatic artifact names, the prompt/payload/snapshot/pricing artifacts, and the shared pipeline used by the web server. -- **Commit** as `refactor(evaluation)!:`/`feat(evaluation)!:` with a `BREAKING CHANGE:` footer saying existing Evaluation run directories are unsupported. - -## Testing Decisions - -- A good test drives the evaluator only through its external seam: the evaluation CLI's `main` entry point, with the LLM request client replaced by the existing fake. Tests assert on the files in the run directory and the text of `report.md`, never on internal helpers. This is the single seam, and it already exists. -- Prior art: the existing evaluation CLI tests already build a small Human audit spreadsheet and HTML input, run live with a fake client, edit `review.csv`, and assert on `report.md`. They also assert that old artifact names are absent after a run. -- Update the existing report assertions to the new rate sentences and names, e.g. LLM detection rate 1/1, Programmatic detection rate 1/1, Overall detection rate 2/3, and the zero-eligible wording. -- New assertions through the same seam: - - The table marks a cross-catch: a `programmatic`-classified finding matched by an LLM finding shows ✓ under LLM without changing the LLM detection rate. - - A missed finding with empty classification reason and review notes is still listed under "Missed". - - Both `unavailable_evidence` and `ambiguous` findings appear under "Not testable from the page file" with their reasons. - - Matched findings show their source label, problem text and "Why this counts". - - The report contains no Human finding IDs, finding IDs, CSS paths or JSON evidence blocks. - - `report` fails without `--reviewer`, and prints it when given. - - The cost line shows cents and the pricing date, and the malformed-output sentence appears only when there are format failures. - - Inline HTML in free text is still escaped. - - After a run, the new artifact names exist and the old ones (`references.json`, `raw-audit-responses.json`, `normalized-audit-findings.json`) do not; review CSV columns use the new names; IDs use the new prefixes. -- No new unit tests below the CLI seam. -- **Live verification (operator-requested, paid):** - 1. After the tests pass, execute one live Evaluation run on the saved Pristine homepage snapshot (`https://pristineai.com/`) with a between-request cost guardrail of $1.10. Previous identical runs cost about $0.11. - 2. A sub-agent acting as the reviewer fills in every `review.csv` row. It classifies against the snapshot independently of the LLM responses, then records Match decisions using the README's rules. - 3. Generate the report with `--reviewer "Claude agent (operator-directed)"` and read it as an outsider would. - -## Out of Scope - -- Changing classification values, scoring rules, review validation rules, or the model ranking decision. -- Migrating, reading or deleting existing local run directories. -- Renaming the Human audit spreadsheet's sheet or columns. -- Changes to the shared pipeline, the web server, or `run_pipeline.py`. -- Precision or other metrics beyond the three detection rates. - -## Further Notes - -- Considered and rejected: showing "unavailable evidence" and "ambiguous" as two report categories. Readers need to know why each finding wasn't counted, and the per-row reasons say that better than category names. The CSV keeps the distinction. -- Considered and rejected: showing ✓/✗ without the tool's text. A ✓ is only believable if the reader can see what the tool said and why it counts. -- Considered and rejected: a migration script or a compatibility layer for old runs. The operator no longer needs them. -- The estimated cost is not an invoice: token counts are exact as reported by the provider, but dollar amounts depend on the recorded public price schedule. - -## Comments diff --git a/.scratch/poc-reduction/spec.md b/.scratch/poc-reduction/spec.md deleted file mode 100644 index c251c26..0000000 --- a/.scratch/poc-reduction/spec.md +++ /dev/null @@ -1,122 +0,0 @@ -# Reduce the accessibility evaluation proof of concept - -Status: resolved - -## Problem Statement - -The evaluation branch adds 8,764 lines and deletes 635 across 61 files relative to `main` at the time of review. It implements substantially more workflow, compatibility, validation, tests, and documentation than the immediate proof of concept needs. The maintainer needs a small, understandable experiment that works on the given Reference workbook and homepage, rather than a reusable evaluation framework. - -The current generated review workbooks also label the source workbook's `element name` as `location`, implying precision the source does not provide. Eligibility and match review require too much machinery, and following an Evaluation run requires understanding file conventions and validation spread across several modules. - -## Solution - -Reduce the POC to one fixed Haiku evaluation using the given Reference workbook and an existing local HTML Benchmark snapshot. Provide a run, human review, report workflow: preview or explicitly approve paid execution, edit one CSV, then generate one Markdown report. Retain Workbook-row recall, programmatic coverage, Combined workbook coverage, Estimated run cost, and traceable evidence. - -Target fewer than 3,000 added lines relative to `main`, counting code, tests, documentation, configuration, and other tracked changes. Reach that target by deleting unsupported scope and repeated coordination, not compressing code or weakening the meaning of the score. - -## User Stories - -1. As a maintainer, I want fewer than 3,000 added lines against `main`, so that I can review and understand the POC. -2. As an evaluator, I want one complete Haiku evaluation, so that I can demonstrate the measurement method without building model-comparison infrastructure. -3. As an evaluator, I want the importer to support the given Reference workbook directly, so that alternative workbook layouts do not complicate this experiment. -4. As an evaluator, I want every applicable Home and Global row retained, so that the denominator does not silently shrink. -5. As a reviewer, I want original defect text and source sheet and row preserved, so that I can trace each Reference defect to its evidence. -6. As a reviewer, I want the original element name displayed as `source_element`, so that it is not mistaken for an inferred precise location. -7. As a reviewer, I want optional `review_notes`, so that I can clarify location or partial coverage without rewriting source evidence. -8. As an evaluator, I want to supply local HTML and its source URL, so that the evaluator does not need a webpage-capture workflow. -9. As an evaluator, I want the snapshot copied into the Evaluation run with its checksum, so that the evaluated input remains identifiable. -10. As an operator, I want a no-cost preview, so that I can inspect the experiment before spending money. -11. As an operator, I want the fixed model configuration and positive cost guardrail displayed before Live-run approval, so that I understand what I am authorizing. -12. As an operator, I want credentials alone to be insufficient for execution, so that an available key cannot trigger accidental spending. -13. As an operator, I want a fresh directory for each Evaluation run, so that new execution does not overwrite earlier evidence. -14. As an operator, I want execution without a separately saved plan, so that I do not manage plan files or verification stages. -15. As an operator, I want failed requests to stop execution without automatic retries, so that failure behavior and additional spending remain simple. -16. As an evaluator, I want responses and reported usage retained when execution fails, so that unsuccessful work remains inspectable and its reported cost is accounted for. -17. As an evaluator, I want incomplete runs excluded from final scoring, so that partial execution cannot appear to be a complete result. -18. As an operator, I want the cost guardrail checked before each paid request, so that execution stops once accumulated reported cost reaches it. -19. As a reviewer, I want one CSV for eligibility and both finding kinds, so that I do not coordinate multiple review workbooks. -20. As a reviewer, I want to edit that CSV in Excel or a text editor, so that human review does not require a custom interface. -21. As a reviewer, I want classifications and Match decisions made by a human, so that the POC does not require an automated Comparison reviewer. -22. As a reviewer, I want to ask Codex to edit decisions on my behalf when needed, so that assistance uses the same review artifact without adding a reviewer subsystem. -23. As a reviewer, I want each row classified as LLM-eligible, programmatic, unavailable evidence, or ambiguous, so that the report distinguishes responsibility and evidence limitations. -24. As a reviewer, I want short reasons for classifications and matches, so that decisions remain explainable. -25. As a reviewer, I want separate audit and programmatic finding-ID columns, so that either or both finding kinds can cover the same Reference defect. -26. As a reviewer, I want stable finding IDs and inspectable finding evidence, so that I can enter defensible matches without candidate-generation machinery. -27. As a reviewer, I want to associate multiple findings with a row when necessary, so that jointly sufficient evidence is representable. -28. As a reviewer, I want to judge full-row coverage myself, so that the evaluator does not need a required-subdefect model. -29. As a reviewer, I want partial coverage documented in notes without automatically awarding credit, so that Workbook-row recall remains a full-row measure. -30. As an evaluator, I want invoking the report command to confirm completion of human review, so that per-row approval states and reviewer metadata are unnecessary. -31. As an evaluator, I want every row classified before reporting, so that unreviewed eligibility cannot change the denominator unnoticed. -32. As an evaluator, I want blank finding-ID cells to mean not caught when reporting, so that misses have a simple representation. -33. As an evaluator, I want invalid or conflicting finding IDs rejected, so that malformed review data cannot silently inflate coverage. -34. As an evaluator, I want deterministic Workbook-row recall over LLM-eligible rows, so that model efficacy is separate from programmatic coverage. -35. As an evaluator, I want programmatic coverage based on reviewed Programmatic findings, so that classification alone does not award credit. -36. As an evaluator, I want Combined workbook coverage to count each source row at most once, so that overlap between finding kinds does not inflate the result. -37. As an evaluator, I want unavailable and ambiguous rows visible in the report, so that exclusions and evidence limitations remain understandable. -38. As an evaluator, I want Estimated run cost derived from reported usage and recorded pricing, so that the experiment's cost is explainable. -39. As a reader, I want one Markdown report with metrics and row-level explanations, so that I can understand the outcome without reconciling report formats. -40. As an evaluator, I want the review CSV, exact prompts, configuration, raw responses, and pricing retained, so that the report has supporting evidence. -41. As an operator, I want existing private artifacts left untouched, so that simplifying the implementation does not destroy previous work. -42. As a maintainer, I want old-format compatibility removed, so that fresh POC runs do not carry historical workflow complexity. -43. As a maintainer, I want the public application's existing provider behavior preserved, so that reducing the evaluator does not regress unrelated functionality. -44. As a maintainer, I want focused tests and a synthetic run-to-report workflow, so that essential behavior is verified without duplicating the implementation. -45. As a maintainer, I want concise current documentation, so that superseded plans and progress narratives do not dominate the change. - -## Implementation Decisions - -- The supported experiment remains the pinned Haiku configuration already implemented: `claude-haiku-4-5-20251001`, 16,000 thinking tokens, 24,192 total output tokens, and omitted temperature. This reduction does not introduce model selection or change audit prompts, filtering, prompt ordering, or disabled summaries. -- Concentrate Evaluation run orchestration behind the existing CLI seam. Expose run and report operations, with no-cost preview and explicit Live-run approval before billable execution. Choose minimal flag spelling during implementation; obsolete workflow options need no compatibility aliases. -- The run operation imports the known Reference workbook, prepares the existing audit pipeline against the local Benchmark snapshot, performs approved execution, and produces evidence and the review CSV. Human review occurs by editing the CSV; reporting reads that run's evidence and decisions. -- Replace generalized workbook import with explicit mapping of the given sheets and columns. Remove header aliases and elaborate input-recovery messages. Ordinary input failures can fail plainly; retain small checks needed to prevent silently omitted or misidentified reference rows. -- Retain source sheet, row, original defect evidence, and the workbook's identity. Use `source_element` for the original element name and optional `review_notes` for reviewer clarification; do not infer locations automatically. -- Use one CSV row per Reference defect with eligibility, short classification and match reasons, and separate `audit_finding_id` and `programmatic_finding_id` fields. Permit multiple IDs within each finding field using one documented delimiter. Handle CSV quoting and multiline text with standard CSV support. -- Retain the four existing eligibility values: `llm_eligible`, `programmatic`, `unavailable_evidence`, and `ambiguous`. Human classifications are required. Codex assistance is an explicit human-directed edit, not an automated reviewer implementation. -- Remove generated Excel review workbooks, candidate-match generation, reviewer protocols for hypothetical future adapters, per-row approval states, reviewer identity, confidence, timestamps, and required-subdefect machinery. Preserve the substantive ADR-0001 decision: review produces decisions; a separate deterministic scorer consumes them. -- Entered finding IDs assert that the selected findings jointly provide full-row coverage. Partial coverage alone belongs in notes and does not earn credit. Reporting constitutes human confirmation that review is complete; blank finding fields then mean not caught. -- Retain validation of required classifications, row identity and completeness, finding existence and kind, and conflicting reuse of findings across reference rows. Allow multiple findings for one reference and allow both finding kinds to cover the same reference. Keep these checks focused on score correctness rather than general spreadsheet robustness. -- Preserve deterministic scoring semantics: Workbook-row recall measures audit coverage of LLM-eligible rows; programmatic classification alone does not count as coverage; Combined workbook coverage counts the union of covered in-scope rows once. Preserve the existing programmatic-coverage denominator and explicitly report unavailable and ambiguous rows. Remove multi-model ranking machinery without changing ADR-0002's ranking policy for future comparisons. -- Accept a local HTML Benchmark snapshot and source URL; copy the HTML into the fresh run directory and record its checksum. Remove evaluator-owned downloading, HTTP metadata validation, atomic snapshot-publication machinery, and capture recovery paths. -- Save workbook identity and imported reference evidence, copied HTML, exact prompts, actual configuration, raw responses, normalized findings, reported usage, pricing, and completion status. Preserve the existing normalization needed by the actual prompt and programmatic outputs; malformed model output must remain visible rather than silently disappearing. -- Remove separately persisted execution plans, plan reloading, repeated cross-module identity checks, compatibility with old artifact formats, workspace-initialization ceremonies, retries, and resume support. Retain basic run/evidence association needed for correct reporting; recorded provenance need not become a general tamper-verification framework. -- Live execution requires credentials, explicit approval of displayed intent, and a positive cost guardrail. Check accumulated reported cost before every request. This is a between-request guardrail, not a guarantee that an in-flight request cannot exceed the remaining amount. -- Stop on a failed paid request or exhausted cost guardrail; preserve evidence and reported usage obtained so far and mark unfinished runs incomplete. Do not automatically retry failed requests. Incomplete runs cannot produce a final coverage score. -- Keep Haiku usage and pricing handling truthful, including reported usage from interrupted streams and billed output used for thinking. Do not fabricate an unavailable reasoning-token breakdown or persist thinking content as audit evidence. -- Produce one Markdown report containing Workbook-row recall, programmatic coverage, Combined workbook coverage, Estimated run cost, execution/format limitations, and row-level explanations. The editable CSV and finding evidence remain supporting artifacts, not additional report projections. -- Preserve existing private artifacts without migrating, overwriting, or deleting them. The simplified workflow requires fresh runs. Keep private inputs and outputs excluded from version control. -- Preserve shared audit-client behavior required by the public application, including existing providers. Remove evaluator-only compatibility without redesigning or duplicating the shared client. -- Consolidate superseded specifications, progress narratives, and implementation records into concise current guidance where needed to meet the size target; earlier content remains in Git history. Preserve applicable agent instructions and existing ADR decisions. -- Measure added lines in the full branch diff from the merge-base with `main`, including deletions of branch-added material and all retained documentation. Do not game the target through minification, dense statements, hidden generated code, or removal of meaningful tests. If essential behavior prevents the target, report the measured gap and tradeoff. - -## Testing Decisions - -- Prefer the existing CLI seam as the primary test surface: exercise preview, run, edit CSV, and report through observable outputs and saved artifacts. No new public testing interface is required. -- Adapt the existing synthetic CLI lifecycle test to the smaller workflow, using a temporary directory, a synthetic workbook with the actual supported headers, local HTML, and the existing fake-provider seam. Verify exact report metrics and traceable row evidence without private data or paid requests. -- Assert externally visible behavior, not helper calls, internal ordering, private data structures, or exact incidental prose. Remove tests for deleted capabilities rather than keeping a second parallel workflow. -- Test source row inclusion and provenance, `source_element`, multiline CSV round-tripping, required eligibility, misses, multiple finding IDs, valid overlap between finding kinds, and rejected unknown or conflicting IDs. Include a case where an automatically populated blank classification cannot produce a report. -- Verify Workbook-row recall, programmatic coverage, and Combined workbook coverage against hand-calculated examples, preserving existing denominator behavior and handling an empty eligible denominator without inventing a score. Use the existing deterministic scoring tests as prior art; retain focused scorer tests only where they express distinct behavior more clearly than the CLI workflow. -- Verify preview and declined approval make no provider calls; credentials alone do not execute. Verify request failure and budget exhaustion retain available evidence and usage, make no automatic retries, and prevent final scoring of incomplete runs. -- Keep focused existing shared-client tests for thinking-stream final text, interrupted-stream usage, and non-evaluation provider behavior. These test an existing external-provider seam rather than creating another evaluator abstraction. -- Run the repository's applicable checks, verify no private evidence is staged, and report full added-line counts against `main`. Tests and specification publication do not authorize paid evaluation. - -## Out of Scope - -- Supporting arbitrary workbook layouts, extensive Excel validation or recovery, generated Excel review workbooks, or a review application. -- Automated AI eligibility or match reviewers, reviewer-adapter frameworks, and per-row approval/audit-trail metadata. -- Multiple-model comparison, ranking implementation, configurable providers for the evaluator, and changes to existing metric policy. -- Automatic location inference, automated full-row coverage judgment, required-subdefect modeling, and partial-credit scoring. -- Webpage capture, browser automation, HTTP provenance frameworks, persisted execution plans, automatic retries, and resumable execution. -- Migration or reporting compatibility for historical evaluation formats; deletion of existing private artifacts. -- Separate JSON or CSV report projections, unrelated public application changes, and changing the audit prompts to fit the evaluator. -- Performing a paid run as part of implementing or verifying this reduction. - -## Further Notes - -This specification supersedes earlier evaluation requirements only where they conflict with the agreed POC reduction. It retains review/scoring separation and Workbook-row recall policy from the existing ADRs; removing future reviewer machinery and ranking code does not replace those policies. - -Inspection found 16 imported Reference defects in the current private artifacts. Their generated eligibility-workbook locations and approved-reference locations matched the imported values, which came from the source `element name` field. This motivates preserving the source label rather than generating a new location. - -The agreed size target is fewer than 3,000 added lines for the whole branch change, not merely the evaluation implementation. The synthetic CLI workflow and fake-provider seam are the primary test surfaces. - -## Comments - -- Implemented in `7b99d1659e29e9b0b50b6b2abff459625bd25781` and `b32eea12d0dbff2db9f7d9ff003abb955547354b`. Verified with 36 passing tests, evaluator typechecking, lint and format checks, and a 2,940-added-line branch diff against `main`. No paid requests or private artifact staging occurred. diff --git a/.scratch/rename-audit-artifacts/spec.md b/.scratch/rename-audit-artifacts/spec.md deleted file mode 100644 index ea6efe2..0000000 --- a/.scratch/rename-audit-artifacts/spec.md +++ /dev/null @@ -1,73 +0,0 @@ -# Name raw and normalized audit artifacts explicitly - -Status: resolved - -## Problem Statement - -An Evaluation run directory stores Programmatic findings under two explicit names, `raw-programmatic-findings.json` and `normalized-programmatic-findings.json`, but stores the AI audit's evidence as `raw-responses.json` and `audit-findings.json`. The two families do not read as parallel: nothing in the audit names says which file is the raw model evidence and which is the normalized set a reviewer copies `finding_id` values from. An evaluator filling in `review.csv` has to consult the README to know which file to open. - -## Solution - -Rename the audit artifacts so that both finding kinds follow the same `raw-…` / `normalized-…` convention, using the glossary term **Audit finding**: - -- `raw-responses.json` becomes `raw-audit-responses.json` -- `audit-findings.json` becomes `normalized-audit-findings.json` - -The raw file keeps the noun "responses" because it holds one entry per prompt, including failed requests, malformed replies, and usage, which are not findings. The normalized file keeps "findings" because it holds Audit findings with `finding_id` values. - -This is a clean break, matching the earlier programmatic rename: new Evaluation runs write only the new names, and `report` reads only the new names. Existing local run directories are discarded rather than migrated. - -## User Stories - -1. As an evaluator, I want the normalized Audit findings file to be named `normalized-audit-findings.json`, so that I can tell at a glance it is the file to copy `audit_finding_id` values from. -2. As an evaluator, I want the raw model evidence file to be named `raw-audit-responses.json`, so that I can tell it holds the unprocessed model replies for the AI audit. -3. As an evaluator, I want the audit and programmatic artifacts to share a `raw-…` / `normalized-…` naming pattern, so that I can find the matching file for either finding kind without reading documentation. -4. As an evaluator, I want the raw audit file to still be called "responses", so that I am not misled into expecting findings for requests that failed or returned malformed output. -5. As an evaluator, I want `report` to read Audit findings from `normalized-audit-findings.json`, so that scoring uses the same file I reviewed. -6. As an evaluator, I want a live Evaluation run to stop writing `raw-responses.json` and `audit-findings.json`, so that a run directory never holds two competing copies of the same evidence. -7. As an evaluator, I want a run stopped by a request failure or the cost guardrail to still record its responses in `raw-audit-responses.json`, so that partial runs remain traceable. -8. As an evaluator, I want `raw-audit-responses.json` and `normalized-audit-findings.json` to be rewritten after every prompt, as the current files are, so that an interrupted run keeps the evidence gathered so far. -9. As an evaluator, I want the README's review instructions to name the new files, so that the steps I follow match what is on disk. -10. As an evaluator, I want the README's run-directory file table to list the raw and normalized files for each finding kind next to each other, so that the parallel structure is visible. -11. As an evaluator, I want the README's troubleshooting guidance to point to `raw-audit-responses.json` when an Audit finding is missing or malformed, so that I look in the right place. -12. As a maintainer, I want the `review.csv` column `audit_finding_id` to stay unchanged, so that the column keeps matching the glossary term Audit finding. -13. As a maintainer, I want the change marked as breaking in its commit, so that anyone holding an old run directory knows `report` will no longer read it. -14. As a maintainer, I want no fallback for the old filenames, so that the evaluator does not carry compatibility code for throwaway local output. -15. As a maintainer, I want the tests to fail if the old filenames are ever written again, so that the rename cannot silently regress. - -## Implementation Decisions - -- The audit execution step that iterates over prompts writes its per-prompt evidence to `raw-audit-responses.json` and its accumulated Audit findings to `normalized-audit-findings.json`. File contents and shapes are unchanged; only the names change. -- The `report` command loads Audit findings from `normalized-audit-findings.json`. Its existing failure when audit evidence is absent (for example, after a preview-only run) is unchanged. -- No compatibility layer: old names are neither read nor written. -- The `review.csv` column `audit_finding_id`, the `run.json` manifest fields (including `format_failures`), and the programmatic artifact names are unchanged. -- `CONTEXT.md` is unchanged. Filenames are an implementation detail, and "Audit finding" remains the correct term. No ADR is needed, since the rename is cheap to reverse. -- README updates cover the review-workflow step that names the finding files, the run-directory file table, the troubleshooting paragraph, the `audit_finding_id` column description, and the list of run artifacts. -- The commit uses the `refactor(evaluation)!:` prefix with a `BREAKING CHANGE:` footer stating that existing Evaluation run directories use unsupported artifact names, following the precedent of the earlier programmatic-findings rename. -- Existing local run directories under the gitignored `.model-evaluation/runs/` are deleted rather than migrated. The rest of `.model-evaluation/` (snapshots, plans) is kept. - -## Testing Decisions - -- Tests exercise external behavior only, through the evaluation CLI's `main` entry point with the audit request client replaced by a fake. That is the single seam, and it already exists. Tests assert on which files exist in the run directory and on `report`'s outcome, not on internal helpers. -- Live-run test: after a live run with a fake client, `raw-audit-responses.json` and `normalized-audit-findings.json` exist; `raw-responses.json` and `audit-findings.json` do not; and `report` succeeds when `review.csv` references `finding_id` values read from `normalized-audit-findings.json`. -- Failure or guardrail test: after a run that stops on the first prompt, `raw-audit-responses.json` holds exactly one response, and `report` still refuses the incomplete run. -- Preview test: unchanged. `report` still fails when no audit artifacts exist. -- Prior art: the existing CLI tests already assert that the old programmatic filenames are absent after a preview run. Mirror that pattern for the old audit filenames. -- No new unit tests at the audit-module level. - -## Out of Scope - -- Renaming the Audit finding concept (for example, to "LLM finding"), the `audit_finding_id` review column, or the `llm_eligible` classification. -- Changing the contents or schema of either audit artifact. -- Migrating or reading old run directories. -- Renaming prompt, payload, snapshot, reference, pricing, or manifest files. -- Any change to the shared pipeline output used by the web server or `run_pipeline.py`. - -## Further Notes - -- Considered and rejected: `raw-llm-responses.json` / `normalized-llm-responses.json`. The `llm` prefix would give one concept a third name alongside "Audit finding" and "AI Audit". The normalized file holds findings, not responses. -- Considered and rejected: `raw-audit-findings.json`. Failed and malformed replies recorded in the raw file contain no findings. - -## Comments - -Implemented in d4a87cf5. Verified with the evaluation CLI tests at the `main` seam: a live run writes `raw-audit-responses.json` and `normalized-audit-findings.json` but not the old names, `report` scores from the normalized file, and a stopped run records one response. Full suite: 36 passed. The two-axis review found no spec gaps. Existing local run directories were deleted. diff --git a/.scratch/run-directory-layout/spec.md b/.scratch/run-directory-layout/spec.md deleted file mode 100644 index d432583..0000000 --- a/.scratch/run-directory-layout/spec.md +++ /dev/null @@ -1,151 +0,0 @@ -# Organize the Evaluation run directory for review and results - -Status: resolved - -## Problem Statement - -A human completing `review.csv` must sift through an Evaluation run directory that mixes review evidence with raw outputs, extracted payloads, prompts, pricing, and canonical Human finding data. Most of those supporting artifacts are unnecessary for ordinary Eligibility classifications and Match decisions. The separate Human finding JSON duplicates evidence already included in the CSV, while the files the reviewer needs are scattered among internal artifacts. - -The same root directory should remain useful after review, when the human reads the scored report. Clearing the root must also preserve access to run completion and malformed-response details, which currently appear in the run manifest and can affect how the reviewer interprets the findings. - -## Solution - -Keep the Evaluation run root focused on review and scored results. Leave the editable CSV, full Benchmark snapshot, both normalized finding lists, and the run manifest at the root. Generate the scored Markdown report there after successful review and reporting. Move supporting artifacts together into one `artifacts/` subdirectory, preserving their filenames. - -The directory contract is: - -| Artifact | Location | Purpose | -| --- | --- | --- | -| `review.csv` | Run root | Original Human finding evidence and editable human decisions | -| `snapshot.html` | Run root | Full Benchmark snapshot used to judge testability and coverage | -| `normalized-llm-findings.json` | Run root | LLM findings with copyable finding IDs | -| `normalized-programmatic-findings.json` | Run root | Programmatic findings with copyable finding IDs | -| `run.json` | Run root | Completion, malformed-response details, configuration, and provenance | -| `report.md` | Run root, after successful reporting | Scored results for the human reader | -| `human-findings.json` | `artifacts/` | Canonical imported Human findings for validation and scoring | -| `pricing.json` | `artifacts/` | Recorded price schedule for Estimated run cost | -| `raw-llm-responses.json` | `artifacts/` | Original LLM replies, usage, and request failures | -| `raw-programmatic-findings.json` | `artifacts/` | Original Programmatic check output | -| `prompts/` | Inside `artifacts/` | Saved prompts used by LLM execution | -| `payloads/` | Inside `artifacts/` | Extracted content used to prepare prompts | - -A complete live run has exactly five files and the supporting directory at its root before reporting. Successful reporting adds the Markdown report. Preview and interrupted runs keep their existing artifact availability and completion semantics, with the same location rules for the artifacts they produce. - -This is a clean break. New runs use only the new layout, and the updated report command reads only that layout. Existing run directories remain untouched and require manual reorganization before the updated report command can read them. - -## User Stories - -1. As a reviewer, I want the Evaluation run root to contain the files needed for review, so that I can identify my working evidence without sorting through internal artifacts. -2. As a reviewer, I want the review CSV to remain at the root, so that I can immediately find the file I must edit. -3. As a reviewer, I want the original Human finding evidence to remain in the CSV, so that I can review each finding without opening a duplicate JSON file. -4. As a reviewer, I want the Human audit's source sheet and row to remain in the CSV, so that I can trace the source evidence. -5. As a reviewer, I want the full Benchmark snapshot at the root, so that I can judge what the saved page can show. -6. As a reviewer, I want the normalized LLM findings at the root, so that I can inspect their evidence when making Match decisions. -7. As a reviewer, I want the normalized Programmatic findings at the root, so that I can inspect the fixed-rule evidence when making Match decisions. -8. As a reviewer, I want finding IDs to remain available in both normalized lists, so that I can copy valid IDs into the review CSV. -9. As a reviewer, I want existing artifact filenames preserved, so that the reorganization does not also require learning new names. -10. As a reviewer, I want the run manifest at the root, so that I can inspect run status before interpreting the findings. -11. As a reviewer, I want incomplete-run reasons preserved, so that I can understand why a run stopped. -12. As a reviewer, I want malformed-response details preserved, so that an empty finding list is not mistaken for evidence that a check found no problems. -13. As a reviewer, I want the scored report at the root after successful reporting, so that I can read the results where I performed the review. -14. As a report reader, I want the report's content unchanged, so that reorganizing files does not change the explanation of the results. -15. As a reviewer, I want supporting artifacts together in one subdirectory, so that they remain discoverable without crowding the root. -16. As a reviewer investigating a result, I want access to the original LLM replies, so that I can inspect missing or malformed output. -17. As a reviewer investigating a result, I want access to raw Programmatic findings, so that I can inspect the original check output. -18. As a reviewer investigating a result, I want access to saved prompts, so that I can see the evidence and instructions sent to the LLM. -19. As a reviewer investigating a result, I want access to extracted payloads, so that I can investigate preprocessing differences from the full Benchmark snapshot. -20. As an evaluator, I want canonical Human finding data retained under supporting artifacts, so that report validation remains reproducible without duplicating everyday reviewer context at the root. -21. As an evaluator, I want the saved price schedule retained under supporting artifacts, so that Estimated run cost remains traceable to its recorded prices. -22. As an operator, I want preview runs to use the new layout, so that file placement is consistent before and after live execution. -23. As an operator, I want preview runs to retain their existing restriction on paid requests, so that reorganizing files does not change Live-run approval. -24. As an operator, I want declined live execution to retain its existing preview evidence and status, so that I can inspect the prepared run. -25. As an operator, I want stopped runs to retain responses and usage in the new locations, so that request failures and exhausted cost guardrails remain inspectable. -26. As an operator, I want evidence saved after each attempted prompt as before, so that an interruption does not lose evidence already gathered. -27. As an evaluator, I want the run's identity to remain the root directory's name, so that the supporting directory is never mistaken for an Evaluation run. -28. As an evaluator, I want normalized findings to retain the correct run identity and source URL, so that evidence validation continues to reject unrelated findings. -29. As an evaluator, I want reporting to read supporting data from its new location, so that I can score reviews of newly generated runs. -30. As an evaluator, I want Eligibility classification and Match decision validation unchanged, so that moving evidence does not alter what counts as coverage. -31. As an evaluator, I want all detection rates unchanged for the same reviewed evidence, so that results remain comparable. -32. As an operator, I want existing run directories left untouched, so that this change does not move, overwrite, or delete prior evidence. -33. As a maintainer, I want the report command to read only the new layout, so that there is one supported artifact contract. -34. As an operator, I want the clean break documented, so that I know an old run needs manual reorganization before reporting. -35. As a maintainer, I want the shared pipeline's output layout outside evaluation preserved, so that the reviewer-focused change does not affect other pipeline users. -36. As a reviewer, I want review and troubleshooting instructions to use the new locations, so that the documented workflow matches generated runs. -37. As a maintainer, I want lifecycle tests to verify the root contents and supporting directory, so that internal artifacts cannot silently reappear at the root. -38. As a maintainer, I want verification through the existing CLI with a fake provider, so that the layout and scoring can be checked without paid execution or new testing interfaces. - -## Implementation Decisions - -- Modify the evaluation CLI's preparation and reporting behavior and the LLM execution module's artifact reads and writes. Retain the existing command arguments and run-directory interface. -- Use the directory contract in the Solution as the externally visible artifact layout. Preserve existing filenames, artifact schemas, CSV columns, finding ID formats, and report content. -- Create the supporting directory before writing or copying artifacts into it. Keep prompts and payloads nested inside that directory. -- Direct the evaluation use of the shared pipeline to the supporting directory for its outputs. Keep the full Benchmark snapshot and normalized Programmatic findings at the root, and preserve the existing cleanup of temporary pipeline output. -- Keep the Evaluation run root distinct from the pipeline output directory. Run manifests and normalized findings must retain the root directory's identity, not the name of the supporting directory. -- LLM execution reads the moved pricing and saved prompts, writes raw responses under supporting artifacts, and continues writing normalized LLM findings and the run manifest at the root. -- Preserve existing per-prompt persistence, response normalization, provider configuration, usage accounting, cost guardrails, and completion behavior. Malformed successful replies retain their current recorded status and normalized-finding behavior. -- Reporting reads canonical Human findings and pricing from supporting artifacts while reading the review CSV, normalized findings, and manifest from the root. Successful reporting writes the report at the root. -- Preserve Human audit identity checks, run identity checks, finding-source checks, source URL checks, review validation, and deterministic scoring. -- Apply the location rules to preview, declined execution, complete execution, and incomplete execution without creating artifacts earlier than their existing lifecycle stage. -- New generation and reporting support only the new layout. Add no old-layout fallback, migration command, or automatic changes to existing runs. -- Update the README's evaluation workflow, artifact table, review instructions, and troubleshooting locations. Explain that existing runs require manual reorganization before the updated report command can read them. -- Preserve the shared pipeline's usual output layout for callers outside evaluation. - -## Testing Decisions - -- Use one existing highest-level seam: the evaluation CLI's `main` entry point, with the existing LLM request client replaced by a fake for live-execution cases. The operator confirmed this lifecycle-testing approach with the complete plan. -- Good tests exercise generated files, retained evidence, CLI outcomes, and report results. They do not assert on internal path helpers, directory-creation call order, or implementation-specific module structure. No new test seam is needed. -- Prior art is the existing evaluation CLI lifecycle suite: it builds a small Human audit and Benchmark snapshot, runs preview or fake live execution, edits the generated CSV, and invokes reporting. It already checks hand-calculated detection rates, review validation, source evidence, run location, and request-failure and cost-guardrail behavior. -- Extend the preview test to check the new locations, preserved source evidence, retained root manifest and normalized Programmatic findings, absent live-only artifacts, no provider calls, and continued refusal to report an incomplete run. -- Extend the successful fake-live lifecycle test to assert the exact pre-report root entries, all supporting entries in their selected locations, and absence of duplicate supporting files at the root. Edit the root CSV using IDs from the root normalized findings, report successfully, and assert that only the report is added to the root. -- Preserve existing report and review-validation assertions, including reviewer attribution and hand-calculated LLM, Programmatic, and Overall detection rates. These verify that reporting can read the moved canonical evidence and price schedule without changing scoring. -- Verify that manifest and finding run IDs still match the run root's name and that existing checks reject evidence belonging to another run. Preserve the existing test for invocation from a worktree subdirectory. -- Extend stopped-run cases to verify raw responses and usage remain available in the selected locations, the root manifest retains the incomplete reason, normalized evidence is retained, and final reporting remains refused. -- Verify malformed successful replies retain their recorded format-failure details at the root and raw responses under supporting artifacts without changing completion semantics or inventing normalized matches. -- Verify per-prompt persistence through the fake provider boundary: before a subsequent request, evidence from the preceding request is already readable at the selected locations. Retain final saved evidence when execution stops. -- Verify reporting has no fallback to old root-level supporting files, using disposable test evidence rather than changing private run directories. Existing directories must not be automatically reorganized. -- Run the evaluation CLI lifecycle suite and the repository's applicable existing checks. Reuse existing shared-pipeline checks to confirm its ordinary output contract remains unchanged. -- No paid live verification is part of this spec. - -## Acceptance Criteria - -- [x] Complete runs expose exactly the agreed review files and `artifacts/` at the root; successful reporting adds only `report.md`. -- [x] Supporting evidence retains its filenames under `artifacts/`, with prompts and payloads nested inside it. -- [x] Preview, declined, stopped, and malformed-response cases preserve artifact availability, approval, completion, usage, and per-prompt persistence behavior. -- [x] Run identity, finding provenance, Human audit identity, review validation, report content, and hand-calculated detection rates remain correct. -- [x] Reporting reads only the new supporting locations and neither falls back to root files nor reorganizes existing evidence. -- [x] The shared pipeline's ordinary output layout remains unchanged outside evaluation. -- [x] README workflow, artifact references, review instructions, and the manual-reorganization requirement match the implemented layout. - -## Out of Scope - -- Generated reviewer guides, readable finding references, or replacement review interfaces. -- New status summaries, CLI warning behavior, filename changes, or artifact schema changes. -- Changes to Eligibility classifications, Match decisions, review validation, detection rates, model ranking, or report content. -- Changes to preprocessing, prompt content, provider configuration, pricing rules, cost guardrails, or Live-run approval. -- Migration commands, compatibility fallbacks, deletion, or automatic reorganization of existing runs. -- Relocating extra files created by historical one-off experiments. -- Changes to the shared pipeline's output layout outside evaluation. -- Paid live execution and implementation of the feature during spec publication. - -## Further Notes - -The CSV already contains the original Human audit evidence; its canonical JSON remains necessary for scorer validation but need not occupy the review-focused root. Normalized findings retain source evidence, while raw replies remain useful for diagnosing malformed LLM output. - -The run manifest deliberately stays visible even though it also contains internal metadata. Moving it would require another way to expose completion and malformed-response status, and the agreed scope is reorganizing existing files. - -Reviewers must continue judging Eligibility classifications against the full Benchmark snapshot. Content omitted from extracted payloads or prompts is not made unavailable merely by that omission. The reorganization does not change this rule. - -The plan preserves the existing decisions to separate human review from deterministic scoring and to rank models by LLM detection rate with Estimated run cost as the tie-breaker. Artifact placement is cheap to reverse and does not warrant an additional ADR or glossary term. - -## Comments - -- Q1: The operator chose to support both review and scored results at the root. -- Q2: The operator chose to reorganize existing files only. -- Q3: The operator chose to keep the run manifest at the root. -- Q4: The operator chose one supporting artifact directory with existing filenames. -- Q5: The operator chose the new layout only, leaving existing directories untouched. -- Q6: The operator confirmed the complete plan and requested publication through the to-spec skill. - -- Implementation complete in commit `1bc12d99` (`feat(evaluation)!: organize supporting run artifacts`). -- Verification: all 47 repository tests passed, including 18 evaluation CLI lifecycle cases. The ordinary shared-pipeline dry run retained its output contract with 10 prompts, 14 Programmatic findings, and zero provider tokens. Lockfile, dependency export, entry-point imports, compilation, and diff checks passed. The repository has no configured type checker. No paid requests or private-run mutations were performed. -- Review: independent Standards and Spec reviews each reported zero findings. All acceptance criteria above are demonstrated by lifecycle tests, the shared-pipeline check, or review. Resolved after the implementation commit and both reviews. diff --git a/README.md b/README.md index 8b10278..bf8f621 100644 --- a/README.md +++ b/README.md @@ -484,11 +484,18 @@ everything it does is free: - `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock - entry points import -- the complete synthetic pytest suite, with no private data or provider calls +- the tracked synthetic pytest suite (`uv run pytest -q`), with no private data or provider calls - a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed - `index.html`'s inline JavaScript parses - the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit +Default pytest discovery is limited to `tests/`. There, `tests/test_evaluation.py` +covers the evaluator end to end through its CLI: preview, live runs against a +fake provider, review validation, scoring, and evidence checks. Detailed +evaluator cases can also be kept in the ignored `.local-tests/evaluation/` +folder and run separately with `uv run pytest -q .local-tests/evaluation`. +That folder is optional, so a fresh checkout and CI never depend on it. + ## Running the Pipeline ### Dry run (no API calls, no cost) diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md index 7ba4a25..cb3ad8a 100644 --- a/docs/agents/issue-tracker.md +++ b/docs/agents/issue-tracker.md @@ -1,6 +1,8 @@ # Issue tracker: Local Markdown -Issues and specs for this repo live as markdown files in `.scratch/`. +Issues and specs for this repo live as markdown files in `.scratch/`. The +folder is ignored by Git: records are the maintainer's local workspace and +are never committed. ## Conventions @@ -13,7 +15,7 @@ Issues and specs for this repo live as markdown files in `.scratch/`. ## Completing implementation issues Close an implementation issue only after its implementation, verification, and -review fixes are committed: +review fixes are committed. The ticket itself stays local: - Change each demonstrated acceptance criterion from `[ ]` to `[x]`. Leave the issue open if any criterion is unmet. @@ -21,7 +23,7 @@ review fixes are committed: triage role from `triage-labels.md`. - Under `## Comments`, append the implementation commit IDs and a concise verification note. -- Commit the ticket update separately so the recorded commit IDs are exact. +- Do not commit the ticket update; `.scratch/` is untracked. ## When a skill says "publish to the issue tracker" diff --git a/pyproject.toml b/pyproject.toml index 3d7ca33..4a5d662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dev = [ [tool.pytest.ini_options] pythonpath = ["."] +testpaths = ["tests"] [tool.setuptools.packages.find] include = ["vision_aid*", "processing_scripts*", "entry_points*"] diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..cf6f88d --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,338 @@ +"""Run, review, and report Evaluation runs through the public CLI.""" + +import csv +from contextlib import nullcontext +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import openpyxl +import pytest + +from processing_scripts.llm.registry import PROMPT_REGISTRY +from vision_aid.evaluation import llm_audit +from vision_aid.evaluation.cli import main + +MODELS = ("claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5") +CATEGORIES = [spec.name for spec in PROMPT_REGISTRY if not spec.is_summary] +REVIEWER = "Test reviewer" +# Synthetic markup that activates every non-summary prompt category. +HTML = ( + 'Home' + '

Home

' + "

News

Year
2026
" + '
' + '

Full name

' + '
Plan' + '
' + '
Team' + 'Donate' + 'SalesLogo' + '
' + "
Footer
" +) + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Build a Human audit and Benchmark snapshot with credentials available.""" + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "Defect report" + sheet.append([ + "Sr. #", "element name", "Browser Combination", "Page name", "Issue Title", + "Steps to Reproduce", "actual result", "expected result", + "Recommendation for Fix", "WCAG Sc", "Type of Change", "comment", + ]) + sheet.append([1, "First link", "Chrome", "Home", "Unclear link", "Open page", + "Vague link\nfor everyone", "Descriptive", "Rename", "2.4.4", "Code", ""]) + for number, title, wcag in ( + (2, "Missing destination", "2.4.4"), (3, "Skipped heading", "1.3.1"), + (4, "Hidden menu", "2.1.1"), (5, "Unclear claim", "4.1.2"), + (6, "No captions", "1.2.2"), + ): + sheet.append([number, "Element", "Chrome", "Global", title, "", "", "", "", wcag]) + sheet.append([7, "Other", "Chrome", "Careers", "Outside scope"]) + book.save(tmp_path / "human-audit.xlsx") + (tmp_path / "home.html").write_text(HTML) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + return tmp_path + + +def run_cli(workspace: Path, *extra: str) -> int: + """Invoke the public run command for the synthetic inputs.""" + return main([ + "run", "--human-audit", str(workspace / "human-audit.xlsx"), + "--html", str(workspace / "home.html"), + "--source-url", "https://example.test/", "--max-cost-usd", "1", *extra, + ]) + + +def report(run: Path) -> str: + """Score a reviewed run and return the Markdown report.""" + assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 + return (run / "report.md").read_text() + + +def refused(run: Path, capsys: pytest.CaptureFixture[str], reviewer: str = REVIEWER) -> str: + """Assert reporting fails without writing a report, returning the error.""" + with pytest.raises(SystemExit) as error: + main(["report", "--run-dir", str(run), "--reviewer", reviewer]) + assert error.value.code == 2 + assert not (run / "report.md").exists() + return capsys.readouterr().err + + +def only_run(workspace: Path) -> Path: + """Find the single run created under the workspace.""" + (run,) = (workspace / ".model-evaluation" / "runs").iterdir() + return run + + +def load(run: Path, name: str): + """Read one saved JSON artifact.""" + return json.loads((run / name).read_text()) + + +def edit_review(run: Path, *decisions: dict[str, str]) -> None: + """Apply reviewer decisions to review.csv rows in Human audit order.""" + path = run / "review.csv" + with path.open(newline="") as stream: + rows = sorted(csv.DictReader(stream), key=lambda row: int(row["source_row"])) + for row, decision in zip(rows, decisions): + row.update(decision) + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=rows[0]) + writer.writeheader() + writer.writerows(reversed(rows)) + + +def fake_client(monkeypatch: pytest.MonkeyPatch, *responses: dict) -> list[str]: + """Replace the request client, replaying responses and recording prompts.""" + prompts: list[str] = [] + default = {"success": True, "response": '[{"problem": "Vague link"}]', + "usage": {"input_tokens": 1, "output_tokens": 1}} + + def call(prompt: str) -> dict: + prompts.append(prompt) + return responses[len(prompts) - 1] if len(prompts) <= len(responses) else default + + monkeypatch.setattr(llm_audit, "LLMRequestClient", + lambda **_: SimpleNamespace(call=call)) + return prompts + + +ROOT = {"review.csv", "snapshot.html", "normalized-programmatic-findings.json", + "run.json", "artifacts"} + + +@pytest.mark.parametrize("model", [None, "claude-opus-5-5", "claude-sonnet-5-5"]) +def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, model): + """Credentials alone never buy requests, and all homepage rows survive.""" + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + assert run_cli(workspace, *(["--model", model] if model else [])) == 0 + run = only_run(workspace) + assert {path.name for path in run.iterdir()} == ROOT + assert {path.name for path in (run / "artifacts").iterdir()} == { + "human-findings.json", "pricing.json", "raw-programmatic-findings.json", + "prompts", "payloads", + } + manifest = load(run, "run.json") + assert manifest["configuration"]["model"] == (model or MODELS[0]) + assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" + assert [prompt["name"] for prompt in manifest["prompts"]] == CATEGORIES + assert (run / "snapshot.html").read_text() == HTML + audit = workspace / "human-audit.xlsx" + assert manifest["human_audit_sha256"] == hashlib.sha256(audit.read_bytes()).hexdigest() + assert manifest["snapshot_sha256"] == hashlib.sha256(HTML.encode()).hexdigest() + raw = load(run, "artifacts/raw-programmatic-findings.json") + normalized = load(run, "normalized-programmatic-findings.json") + assert raw and [item["raw_source"] for item in normalized] == raw + assert all(item["run_id"] == run.name for item in normalized) + with (run / "review.csv").open(newline="") as stream: + rows = list(csv.DictReader(stream)) + assert [row["source_row"] for row in rows] == ["2", "3", "4", "5", "6", "7"] + assert rows[0]["actual result"] == "Vague link\nfor everyone" + assert all(not row["classification"] for row in rows) + + +def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( + workspace, monkeypatch, capsys, +): + """Declined approval prepares a preview; invalid models create no run.""" + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call")) + with pytest.raises(SystemExit) as error: + run_cli(workspace, "--model", "gpt-4o", "--live", "--approve-live") + assert error.value.code == 2 and "invalid choice" in capsys.readouterr().err + assert not (workspace / ".model-evaluation").exists() + monkeypatch.setattr("sys.stdin", SimpleNamespace(isatty=lambda: False)) + assert run_cli(workspace, "--live") == 0 + manifest = load(only_run(workspace), "run.json") + assert manifest["incomplete_reason"] == "preview" + assert not (only_run(workspace) / "artifacts" / "raw-llm-responses.json").exists() + + +@pytest.mark.parametrize("model,thinking,effort,cost", [ + (MODELS[0], {"type": "enabled", "budget_tokens": 16000}, None, "0.03915"), + (MODELS[1], {"type": "adaptive"}, "medium", "0.1548"), + (MODELS[2], {"type": "adaptive"}, "medium", "0.0783"), +]) +def test_presets_stream_every_category_and_report_saved_prices( + workspace, monkeypatch, model, thinking, effort, cost, +): + """Each preset's request settings, final text, model, and costs persist.""" + requests, options = [], [] + + def create(**kwargs): + requests.append(kwargs) + usage = SimpleNamespace(input_tokens=1000, output_tokens=1, + cache_read_input_tokens=500, cache_creation_input_tokens=100) + return nullcontext(iter([ + SimpleNamespace(type="message_start", message=SimpleNamespace( + id="synthetic", model=model, type="message", usage=usage)), + SimpleNamespace(type="content_block_start", + content_block=SimpleNamespace(type="thinking", thinking="SECRET")), + SimpleNamespace(type="content_block_start", content_block=SimpleNamespace( + type="text", text='[{"problem": "Synthetic failure", "location": "body"}]')), + SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), + usage=SimpleNamespace(output_tokens=200)), + SimpleNamespace(type="message_stop"), + ])) + + def client(**kwargs): + options.append(kwargs) + return SimpleNamespace(messages=SimpleNamespace(create=create)) + + monkeypatch.setattr("anthropic.Anthropic", client) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert options == [{"api_key": "synthetic", "max_retries": 0}] + assert len(requests) == len(CATEGORIES) + for request in requests: + assert request["model"] == model and request["thinking"] == thinking + assert request["max_tokens"] == 24192 and request["stream"] is True + assert "temperature" not in request + assert request.get("output_config") == ({"effort": effort} if effort else None) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["estimated_cost_usd"] == cost + assert manifest["configuration"]["reasoning_effort"] == effort + raw = load(run, "artifacts/raw-llm-responses.json") + assert all(item["result"]["provider"]["response_model"] == model for item in raw) + assert "SECRET" not in json.dumps(raw) + findings = load(run, "normalized-llm-findings.json") + assert [finding["prompt"] for finding in findings] == CATEGORIES + assert all(finding["model"] == model for finding in findings) + assert all(finding["problem"] == "Synthetic failure" for finding in findings) + assert all(finding["finding_id"].startswith("llm-") for finding in findings) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + text = report(run) + assert model in text and run.name in text + assert "published prices captured on 2026-10-01" in text + assert "18000 input tokens" in text and "3600 output tokens" in text + assert "9000 cached input tokens" in text and "1800 cache-creation input tokens" in text + + +def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): + """Known decisions yield hand-calculated rates, cross-catches, and exclusions.""" + fake_client( + monkeypatch, + {"success": True, "response": "not JSON", "usage": {}}, + {"success": True, "response": "[]", "usage": {}}, + ) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["format_failures"] == ["page_title"] + llm = [item["finding_id"] for item in load(run, "normalized-llm-findings.json")] + assert len(llm) == len(CATEGORIES) - 2 + programmatic = load(run, "normalized-programmatic-findings.json")[0]["finding_id"] + assert "classification" in refused(run, capsys) + decisions = [ + {"classification": "llm_eligible", "llm_finding_id": llm[0], "match_reason": "Full issue"}, + {"classification": "programmatic", "programmatic_finding_id": programmatic, + "match_reason": "HTML check"}, + {"classification": "programmatic", "llm_finding_id": llm[1], "match_reason": "Cross-catch"}, + {"classification": "unavailable_evidence", "classification_reason": "Needs a browser"}, + {"classification": "ambiguous", "classification_reason": "Needs judgment"}, + {"classification": "llm_eligible", "review_notes": "No full finding"}, + ] + edit_review(run, *decisions) + refused(run, capsys, reviewer=" ") + edit_review(run, {"match_reason": ""}) + assert "needs a match reason" in refused(run, capsys) + edit_review(run, {"llm_finding_id": programmatic, "match_reason": "Full issue"}) + assert "Unknown LLM finding ID" in refused(run, capsys) + edit_review(run, *decisions) + text = report(run) + assert ("Of the 2 Human findings the LLM could be expected to detect from the " + "page's HTML, it caught 1 (50.0%)") in text + assert "they caught 1 (50.0%)" in text + assert "Of all 6 Human findings, either part of the tool caught 3 (50.0%)" in text + assert "| Skipped heading | 1.3.1 | Programmatic checks | ✓ | ✗ |" in text + assert "Why this counts: Full issue" in text + missed = text.split("## Missed\n", 1)[1].split("## Not testable", 1)[0] + assert "### No captions" in missed and "No full finding" in missed + excluded = text.split("## Not testable from the page file\n", 1)[1] + assert "### Hidden menu" in excluded and "Needs a browser" in excluded + assert "### Unclear claim" in excluded and "Needs judgment" in excluded + assert "The LLM returned unreadable output for 1 of 18 checks: page title check." in text + assert all(finding_id not in text for finding_id in [*llm, programmatic]) + edit_review(run, *[{"classification": "ambiguous", "llm_finding_id": "", + "programmatic_finding_id": "", "match_reason": ""}] * 6) + text = report(run) + assert "No Human findings were expected to be detected by the LLM" in text + assert "No Human findings were expected to be detected by the Programmatic checks" in text + assert "Of all 6 Human findings, either part of the tool caught 0 (0.0%)" in text + + +@pytest.mark.parametrize("failure,calls,cost,reason", [ + (True, 2, "0.2002", "synthetic failure"), + (False, 1, "0.1001", "cost guardrail exhausted"), +]) +def test_failure_or_exhaustion_stops_and_keeps_partial_evidence( + workspace, monkeypatch, capsys, failure, calls, cost, reason, +): + """Requests stop, billed usage and findings remain, and reporting refuses.""" + success = {"success": True, "response": '[{"problem": "Unclear title"}]', + "usage": {"input_tokens": 100000, "output_tokens": 20}} + stopped = {"success": False, "response": None, "error": "synthetic failure", + "usage": {"input_tokens": 100000, "output_tokens": 20}} + prompts = fake_client(monkeypatch, success, stopped if failure else success) + limit = "1" if failure else "0.01" + assert run_cli(workspace, "--max-cost-usd", limit, "--live", "--approve-live") == 1 + assert len(prompts) == calls + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is False and manifest["incomplete_reason"] == reason + assert manifest["usage"]["input_tokens"] == calls * 100000 + assert manifest["estimated_cost_usd"] == cost + assert len(load(run, "artifacts/raw-llm-responses.json")) == calls + assert len(load(run, "normalized-llm-findings.json")) == 1 + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + assert "Incomplete" in refused(run, capsys) + + +@pytest.mark.parametrize("name,field,message", [ + ("run.json", "run_id", "does not belong"), + ("normalized-llm-findings.json", "run_id", "do not belong"), + ("normalized-programmatic-findings.json", "page_url", "do not belong"), + ("artifacts/human-findings.json", "human_audit_sha256", "identity differs"), + ("artifacts/pricing.json", None, "pricing.json"), +]) +def test_report_rejects_unrelated_or_missing_evidence( + workspace, monkeypatch, capsys, name, field, message, +): + """Reports cannot combine evidence from different or partial runs.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + if field is None: + (run / name).unlink() + else: + evidence = load(run, name) + (evidence[0] if isinstance(evidence, list) else evidence)[field] = "unrelated" + (run / name).write_text(json.dumps(evidence)) + assert message in refused(run, capsys) diff --git a/tests/test_evaluation_cli.py b/tests/test_evaluation_cli.py deleted file mode 100644 index f198287..0000000 --- a/tests/test_evaluation_cli.py +++ /dev/null @@ -1,762 +0,0 @@ -"""Run, review, and report through the CLI and its saved artifacts.""" - -import csv -from contextlib import nullcontext -import hashlib -import json -from pathlib import Path -from types import SimpleNamespace - -import openpyxl -import pytest - -from vision_aid.evaluation import llm_audit -from vision_aid.evaluation.cli import main - - -def inputs(tmp_path: Path) -> tuple[Path, Path]: - """Build a small Human audit with the exact supported source columns.""" - book = openpyxl.Workbook() - sheet = book.active - sheet.title = "Defect report" - sheet.append( - [ - "Sr. #", - "element name", - "Browser Combination", - "Page name", - "Issue Title", - "Steps to Reproduce", - "actual result", - "expected result", - "Recommendation for Fix", - "WCAG Sc", - "Type of Change", - "comment", - ] - ) - for row in ( - [1, "First link", "Chrome", "Home", "Unclear link", "Open page", - "Vague link\nfor everyone", "Descriptive text", "Rename", "2.4.4", "Code", ""], - [2, "Second link", "Chrome", "Global", "Missing destination", "Open page", - "No href", "Working link", "Add href", "2.4.4", "Code", ""], - [3, "Photo", "Chrome", "Home", "No alt text", "Open page", - "Image lacks alt", "Alt text", "Add alt", "1.1.1", "Code", ""], - ): - sheet.append(row) - sheet.append([4, "Other", "Chrome", "Careers", "Outside scope"]) - human_audit = tmp_path / "human-audit.xlsx" - book.save(human_audit) - html = tmp_path / "home.html" - html.write_text( - 'Home' - '

Home

Click here' - ) - return human_audit, html - - -def args(human_audit: Path, html: Path, *extra: str) -> list[str]: - """Assemble the public run command for the synthetic inputs.""" - return [ - "run", - "--human-audit", - str(human_audit), - "--html", - str(html), - "--source-url", - "https://example.test/", - "--max-cost-usd", - "0.01", - *extra, - ] - - -def directory(tmp_path: Path) -> Path: - """Find the single run created under a temporary workspace.""" - return next((tmp_path / ".model-evaluation" / "runs").iterdir()) - - -def read_review(path: Path) -> list[dict[str, str]]: - """Read reviewer decisions using standard CSV quoting.""" - with path.open(newline="") as stream: - return list(csv.DictReader(stream)) - - -def write_review(path: Path, rows: list[dict[str, str]]) -> None: - """Save edited reviewer decisions using standard CSV quoting.""" - with path.open("w", newline="") as stream: - writer = csv.DictWriter(stream, fieldnames=rows[0]) - writer.writeheader() - writer.writerows(rows) - - -@pytest.mark.parametrize("model,thinking,effort,input_rate,output_rate", [ - ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "1", "5"), - ("claude-opus-5-5", {"type": "adaptive"}, "medium", "4", "20"), - ("claude-sonnet-5-5", {"type": "adaptive"}, "medium", "2", "10"), -]) -def test_selected_model_preview_records_preset_without_provider_access( - tmp_path, monkeypatch, capsys, model, thinking, effort, input_rate, output_rate, -): - """A key-free preview records and displays the selected preset and prices.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) - - assert main(args(human_audit, html, "--model", model)) == 0 - run = directory(tmp_path) - manifest = json.loads((run / "run.json").read_text()) - config = manifest["configuration"] - assert config["model"] == model - assert config["thinking"] == thinking - assert config["reasoning_effort"] == effort - assert config["max_output_tokens"] == 24192 - assert config["temperature"] == "omitted" - assert config["summaries"] == "disabled" - assert config["provider"] == "anthropic" and config["endpoint"] == "messages" - schedule = json.loads((run / "artifacts" / "pricing.json").read_text()) - assert schedule["models"][model]["input_per_million_usd"] == input_rate - assert schedule["models"][model]["output_per_million_usd"] == output_rate - assert manifest["pricing_sha256"] == hashlib.sha256((run / "artifacts" / "pricing.json").read_bytes()).hexdigest() - assert not (run / "artifacts" / "raw-llm-responses.json").exists() - output = capsys.readouterr().out - assert model in output and "24192" in output - assert f"input ${input_rate}; output ${output_rate}" in output - assert "16000" in output if effort is None else "adaptive; effort: medium" in output - assert "request underway can exceed" in output - - -@pytest.mark.parametrize("model", ["claude-haiku-4-5", "claude-unknown", "gpt-4o", "opus"]) -def test_unsupported_model_is_rejected_without_run_or_provider_access( - tmp_path, monkeypatch, capsys, model, -): - """Unsupported IDs and aliases fail before creating evidence or buying requests.""" - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) - with pytest.raises(SystemExit) as error: - main(args(tmp_path / "missing.xlsx", tmp_path / "missing.html", "--model", model, "--live", "--approve-live")) - assert error.value.code == 2 - message = capsys.readouterr().err - assert "invalid choice" in message - assert all(supported in message for supported in ( - "claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5", - )) - assert not (tmp_path / ".model-evaluation").exists() - - -@pytest.mark.parametrize("model,thinking,effort,cost", [ - ("claude-haiku-4-5-20251001", {"type": "enabled", "budget_tokens": 16000}, None, "0.006525"), - ("claude-opus-5-5", {"type": "adaptive"}, "medium", "0.0258"), - ("claude-sonnet-5-5", {"type": "adaptive"}, "medium", "0.01305"), -]) -def test_selected_model_executes_and_reports_with_saved_prices( - tmp_path, monkeypatch, model, thinking, effort, cost, -): - """The real CLI and request client retain selected-model evidence and costs.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - requests = [] - client_options = [] - - def create(**kwargs): - requests.append(kwargs) - assert kwargs["model"] == model - assert kwargs["thinking"] == thinking - assert kwargs["max_tokens"] == 24192 and kwargs["stream"] is True - assert "temperature" not in kwargs - if effort: - assert kwargs["output_config"] == {"effort": "medium"} - else: - assert "output_config" not in kwargs - return nullcontext(iter([ - SimpleNamespace(type="message_start", message=SimpleNamespace( - id="synthetic-response", model=model, type="message", - usage=SimpleNamespace(input_tokens=1000, output_tokens=1, - cache_read_input_tokens=500, cache_creation_input_tokens=100), - )), - SimpleNamespace(type="content_block_start", content_block=SimpleNamespace(type="thinking", thinking="SECRET")), - SimpleNamespace(type="content_block_delta", delta=SimpleNamespace(type="thinking_delta", thinking="SECRET")), - SimpleNamespace(type="content_block_start", content_block=SimpleNamespace(type="text", text='[{"problem": "Unclear link"}]')), - SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), usage=SimpleNamespace(output_tokens=200)), - SimpleNamespace(type="message_stop"), - ])) - - def client(**options): - client_options.append(options) - return SimpleNamespace(messages=SimpleNamespace(create=create)) - - monkeypatch.setattr("anthropic.Anthropic", client) - assert main(args(human_audit, html, "--model", model, "--max-cost-usd", "1", "--live", "--approve-live")) == 0 - assert client_options == [{"api_key": "synthetic", "max_retries": 0}] - assert len(requests) == 3 - run = directory(tmp_path) - manifest = json.loads((run / "run.json").read_text()) - assert manifest["configuration"]["model"] == model - assert manifest["configuration"]["thinking"] == thinking - assert manifest["configuration"]["reasoning_effort"] == effort - assert manifest["estimated_cost_usd"] == cost - findings = json.loads((run / "normalized-llm-findings.json").read_text()) - assert len(findings) == 3 and all(finding["model"] == model for finding in findings) - raw = json.loads((run / "artifacts" / "raw-llm-responses.json").read_text()) - assert all(item["result"]["provider"]["response_model"] == model for item in raw) - assert all(item["result"]["request"]["thinking"] == thinking for item in raw) - assert "SECRET" not in json.dumps(raw) - rows = read_review(run / "review.csv") - for row in rows: - row.update(classification="llm_eligible") - write_review(run / "review.csv", rows) - assert main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) == 0 - report = (run / "report.md").read_text() - assert model in report - assert "published prices captured on 2026-10-01" in report - - -def test_preview_preserves_human_findings_and_never_calls_provider( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Credentials alone never buy requests, and all source rows survive.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "available") - monkeypatch.setattr( - llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call") - ) - assert main(args(human_audit, html)) == 0 - output = capsys.readouterr().out - run = directory(tmp_path) - assert {path.name for path in run.iterdir()} == { - "review.csv", "snapshot.html", "normalized-programmatic-findings.json", - "run.json", "artifacts", - } - artifacts = run / "artifacts" - assert {path.name for path in artifacts.iterdir()} == { - "human-findings.json", "pricing.json", "raw-programmatic-findings.json", - "prompts", "payloads", - } - assert list((artifacts / "prompts").glob("*.json")) - assert {path.name for path in (artifacts / "payloads").iterdir()} == { - "cl01_payload.json", "cl02_payload.json", "cl03_payload.json", - } - assert "claude-haiku-4-5-20251001" in output - assert "16000" in output and "24192" in output and "$0.01" in output - assert (run / "snapshot.html").read_bytes() == html.read_bytes() - raw = json.loads((run / "artifacts" / "raw-programmatic-findings.json").read_text()) - normalized = json.loads((run / "normalized-programmatic-findings.json").read_text()) - assert raw and [finding["raw_source"] for finding in normalized] == raw - assert not (run / "programmatic_findings.json").exists() - assert not (run / "programmatic-findings.json").exists() - manifest = json.loads((run / "run.json").read_text()) - assert manifest["run_id"] == run.name - assert all(item["run_id"] == run.name for item in normalized) - assert manifest["snapshot_sha256"] == hashlib.sha256(html.read_bytes()).hexdigest() - assert manifest["human_audit_filename"] == human_audit.name - assert manifest["human_audit_sha256"] == hashlib.sha256(human_audit.read_bytes()).hexdigest() - imported = json.loads((run / "artifacts" / "human-findings.json").read_text()) - assert imported["human_audit_sha256"] == manifest["human_audit_sha256"] - assert all(item["human_finding_id"].startswith("human-") for item in imported["human_findings"]) - rows = read_review(run / "review.csv") - assert [row["source_row"] for row in rows] == ["2", "3", "4"] - assert rows[0]["source_element"] == "First link" - assert rows[0]["actual result"] == "Vague link\nfor everyone" - assert all(not row["classification"] for row in rows) - assert {"human_finding_id", "llm_finding_id"} <= rows[0].keys() - assert not {"reference_id", "audit_finding_id"} & rows[0].keys() - with pytest.raises(SystemExit) as help_exit: - main(["run", "--help"]) - assert help_exit.value.code == 0 - help_text = " ".join(capsys.readouterr().out.split()) - assert "Human audit" in help_text - assert "16,000" in help_text and "24,192" in help_text - assert "adaptive thinking with medium effort" in help_text - assert "a request underway can exceed" in help_text - with pytest.raises(SystemExit): - main([*args(human_audit, html), "--workbook", str(human_audit)]) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - - -def test_subdirectory_invocation_uses_ignored_worktree_run_root( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A run from a repository subdirectory stays under the root ignore rule.""" - human_audit, html = inputs(tmp_path) - (tmp_path / ".git").mkdir() - child = tmp_path / "entry_points" - child.mkdir() - monkeypatch.chdir(child) - assert main(args(human_audit, html)) == 0 - assert (tmp_path / ".model-evaluation" / "runs").is_dir() - assert not (child / ".model-evaluation").exists() - - -def test_live_csv_review_and_report_full_row_metrics( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """One fake live run yields hand-calculated metrics and source evidence.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - calls = 0 - - def call(self, prompt: str) -> dict: - """Return a successful fake provider response with usage.""" - if self.calls: - saved = directory(tmp_path) - responses = json.loads((saved / "artifacts" / "raw-llm-responses.json").read_text()) - findings = json.loads((saved / "normalized-llm-findings.json").read_text()) - manifest = json.loads((saved / "run.json").read_text()) - assert len(responses) == self.calls - assert len(findings) == self.calls - assert manifest["usage"]["input_tokens"] == self.calls * 1000 - assert manifest["complete"] is False - self.calls += 1 - return { - "success": True, - "response": json.dumps( - [{"problem": "Unclear link", "location": "First link"}] - ), - "usage": { - "input_tokens": 1000, - "output_tokens": 200, - "cached_input_tokens": 500, - "cache_creation_input_tokens": 100, - "reasoning_tokens": None, - }, - } - - fake = FakeClient() - client_options: list[dict] = [] - - def client(**options: object) -> FakeClient: - """Record the evaluator's request-client configuration.""" - client_options.append(options) - return fake - - monkeypatch.setattr(llm_audit, "LLMRequestClient", client) - assert main(args(human_audit, html, "--live", "--approve-live")) == 0 - assert client_options[0]["max_retries"] == 0 - run = directory(tmp_path) - assert fake.calls == 3 - root_entries = { - "review.csv", "snapshot.html", "normalized-llm-findings.json", - "normalized-programmatic-findings.json", "run.json", "artifacts", - } - assert {path.name for path in run.iterdir()} == root_entries - assert {path.name for path in (run / "artifacts").iterdir()} == { - "human-findings.json", "pricing.json", "raw-llm-responses.json", - "raw-programmatic-findings.json", "prompts", "payloads", - } - assert json.loads((run / "run.json").read_text())["run_id"] == run.name - assert ( - json.loads((run / "run.json").read_text())["estimated_cost_usd"] == "0.006525" - ) - assert (run / "artifacts" / "raw-llm-responses.json").exists() - assert all(not (run / name).exists() for name in ( - "references.json", "raw-audit-responses.json", "normalized-audit-findings.json" - )) - findings = json.loads((run / "normalized-llm-findings.json").read_text()) - programmatic_file = run / "normalized-programmatic-findings.json" - programmatic = json.loads(programmatic_file.read_text()) - assert findings and programmatic - assert all(item["run_id"] == run.name for item in findings + programmatic) - assert all(item["source"] == "llm" and item["finding_id"].startswith("llm-") for item in findings) - assert all(item["finding_id"].startswith("programmatic-") for item in programmatic) - review = run / "review.csv" - rows = read_review(review) - rows[0].update( - classification="llm_eligible", - classification_reason="Model can see link", - llm_finding_id=findings[-1]["finding_id"], - match_reason="Full issue", - ) - rows[1].update( - classification="programmatic", - classification_reason="HTML check", - programmatic_finding_id=programmatic[0]["finding_id"], - match_reason="Full issue", - ) - rows[2].update( - classification="unavailable_evidence", classification_reason="Visual context" - ) - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run)]) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", " "]) - write_review(review, list(reversed(rows))) - # Reproduce saved evidence from before selectable model presets existed. - old_manifest = json.loads((run / "run.json").read_text()) - old_manifest["configuration"].pop("thinking") - old_manifest["configuration"].pop("reasoning_effort") - old_pricing = { - "models": {"claude-haiku-4-5-20251001": { - "input_per_million_usd": "1", "output_per_million_usd": "5", - "cached_input_per_million_usd": "0.10", "cache_creation_input_per_million_usd": "1.25", - }}, - "published_at": "2026-09-22", - "source": "https://platform.claude.com/docs/en/about-claude/pricing", - "version": "anthropic-public-2026-09-22", - } - pricing_path = run / "artifacts" / "pricing.json" - pricing_path.write_text(json.dumps(old_pricing)) - old_manifest["pricing_version"] = old_pricing["version"] - old_manifest["pricing_sha256"] = hashlib.sha256(pricing_path.read_bytes()).hexdigest() - (run / "run.json").write_text(json.dumps(old_manifest)) - assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 - assert {path.name for path in run.iterdir()} == root_entries | {"report.md"} - report = (run / "report.md").read_text() - assert "# LLM accessibility evaluation: https://example.test/" in report - headings = ["How to read this report", "Summary", "Human findings", "What each part of the tool caught", "Missed", "Not testable from the page file"] - assert [report.index("## " + heading + "\n") for heading in headings] == sorted(report.index("## " + heading + "\n") for heading in headings) - assert all(text in report for text in ( - "benchmark is the Human audit", "This API uses LLM checks and Programmatic checks", - "send extracted page content to the model", "apply fixed rules directly", "only saved HTML", "homepage only", "A reviewer decided", - "fully cover the whole Human finding", "Partial coverage earns no credit", - )) - assert "Review decisions by: Codex agent (operator-directed)" in report - assert "claude-haiku-4-5-20251001" in report and run.name in report - assert "Of the 1 Human findings the LLM could be expected to detect from the page's HTML, it caught 1 (100.0%)" in report - assert "Programmatic detection rate: Of the 1 Human findings the Programmatic checks could be expected to detect from the page's HTML, they caught 1 (100.0%)" in report - assert "Of all 3 Human findings, either part of the tool caught 2 (66.7%)" in report - assert "Estimated run cost: $0.01" in report - assert "3000 input tokens" in report and "600 output tokens" in report - assert "1500 cached input tokens" in report and "300 cache-creation input tokens" in report - assert "2026-09-22" in report and "The actual bill may differ slightly." in report - assert "unreadable output" not in report - table = report.split("## Human findings\n", 1)[1].split("## What each part", 1)[0] - assert table.index("Unclear link") < table.index("Missing destination") < table.index("No alt text") - assert "| Unclear link | 2.4.4 | LLM | ✓ | ✗ |" in table - assert "| Missing destination | 2.4.4 | Programmatic checks | ✗ | ✓ |" in table - assert "LLM (link clarity check): Unclear link" in report - assert "Programmatic check " + programmatic[0]["prompt"].replace("_", r"\_") in report - assert "\n\n - Why this counts: Full issue" in report - assert "Visual context" in report - assert all(row["human_finding_id"] not in report for row in rows) - assert all(item["finding_id"] not in report for item in findings + programmatic) - assert not any(text in report for text in ("source_element", "First link", "```", "
", "reasoning_tokens", "@ ")) - for row in rows: - row["audit_finding_id"] = row.pop("llm_finding_id") - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - imported = json.loads((run / "artifacts" / "human-findings.json").read_text()) - imported["human_audit_sha256"] = "changed" - (run / "artifacts" / "human-findings.json").write_text(json.dumps(imported)) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - - -def test_report_cross_catches_empty_misses_and_not_testable_findings( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Cross-catches count overall, blank misses survive, and text is escaped.""" - human_audit, html = inputs(tmp_path) - book = openpyxl.load_workbook(human_audit) - book.active["E2"] = "Unclear
|link" - book.active.append([5, "Menu", "Chrome", "Home", "Hidden menu", "", "", "", "", "2.1.1"]) - book.active.append([6, "Widget", "Chrome", "Home", "Unclear claim", "", "", "", "", "4.1.2"]) - book.save(human_audit) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - calls = 0 - - def call(self, prompt: str) -> dict: - """One unreadable response and two findings with private locations.""" - self.calls += 1 - return { - "success": True, - "response": "not JSON" if self.calls == 1 else json.dumps([{"problem": "Use
|landmark", "location": "body > main"}]), - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - - fake = FakeClient() - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: fake) - assert main(args(human_audit, html, "--live", "--approve-live")) == 0 - run = directory(tmp_path) - findings = json.loads((run / "normalized-llm-findings.json").read_text()) - manifest = json.loads((run / "run.json").read_text()) - responses = json.loads((run / "artifacts" / "raw-llm-responses.json").read_text()) - assert manifest["complete"] is True - assert manifest["format_failures"] == ["page_title"] - assert responses[0]["result"]["response"] == "not JSON" - assert len(findings) == 2 - assert all(item["prompt"] != "page_title" for item in findings) - review = run / "review.csv" - rows = read_review(review) - rows[0].update(classification="llm_eligible", llm_finding_id=findings[0]["finding_id"], match_reason="Full
|coverage") - rows[1].update(classification="programmatic", llm_finding_id=findings[1]["finding_id"], match_reason="Cross-catch") - rows[2].update(classification="programmatic") - rows[3].update(classification="unavailable_evidence", classification_reason="Requires a live browser") - rows[4].update(classification="ambiguous", classification_reason='An |needs judgment') - write_review(review, list(reversed(rows))) - assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 - report = (run / "report.md").read_text() - assert r"Unclear \\|link" in report - assert r"Use \\|landmark" in report - assert "\n\n - " + r"Why this counts: Full \\|coverage" in report - assert "| Missing destination | 2.4.4 | Programmatic checks | ✓ | ✗ |" in report - assert "it caught 1 (100.0%)" in report and "they caught 0 (0.0%)" in report - assert "Of all 5 Human findings, either part of the tool caught 2 (40.0%)" in report - assert "count only catches by the expected part" in report - missed = report.split("## Missed\n", 1)[1].split("## Not testable", 1)[0] - assert "### No alt text" in missed - assert "Why it was expected to be caught" not in missed and "Why it was missed" not in missed - excluded = report.split("## Not testable from the page file\n", 1)[1] - assert "### Hidden menu" in excluded and "Requires a live browser" in excluded - assert "### Unclear claim" in excluded and r'An \\|needs judgment' in excluded - assert "remain in the Overall denominator" in excluded - assert "3 input tokens" in report and "3 output tokens" in report - assert "cached input" not in report and "cache-creation" not in report - assert "The LLM returned unreadable output for 1 of 3 checks: page title check." in report - assert all(row["human_finding_id"] not in report for row in rows) - assert all(item["finding_id"] not in report for item in findings) - assert not any(text in report for text in ("
", '', "body > main", "```", "source_element")) - rows[2].update(classification_reason="The image is in the HTML", review_notes="No full finding") - write_review(review, rows) - assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 - explained = (run / "report.md").read_text().split("## Missed\n", 1)[1] - assert "\n- Why it was expected to be caught: The image is in the HTML" in explained - assert "\n- Why it was missed: No full finding" in explained - - -def test_review_requires_classification_and_rejects_invalid_finding_ids( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """Review validation blocks missing decisions, wrong kinds, and reuse.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - def call(self, prompt: str) -> dict: - """Return one normalized finding for applicable prompts.""" - return { - "success": True, - "response": json.dumps([{"problem": "Vague link"}]), - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) - assert main(args(human_audit, html, "--live", "--approve-live")) == 0 - run = directory(tmp_path) - review = run / "review.csv" - llm_ids = [ - item["finding_id"] - for item in json.loads((run / "normalized-llm-findings.json").read_text()) - ] - programmatic_id = json.loads( - (run / "normalized-programmatic-findings.json").read_text() - )[0]["finding_id"] - rows = read_review(review) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - rows[0].update( - classification="llm_eligible", - classification_reason="Visible", - llm_finding_id="; ".join(llm_ids[:2]), - programmatic_finding_id=programmatic_id, - match_reason="Together cover the row", - review_notes="Partial alone", - ) - rows[1].update(classification="programmatic", classification_reason="Markup") - rows[2].update(classification="ambiguous", classification_reason="Unclear claim") - write_review(review, rows) - assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 - assert "Of all 3 Human findings, either part of the tool caught 1 (33.3%)" in (run / "report.md").read_text() - rows[0]["match_reason"] = "" - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - assert "needs a match reason" in capsys.readouterr().err - rows[0]["match_reason"] = "Together cover the row" - rows[1]["llm_finding_id"] = llm_ids[0] - rows[1]["match_reason"] = "Conflicts" - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - assert "Conflicting LLM finding ID" in capsys.readouterr().err - rows[1]["llm_finding_id"] = "llm-unknown" - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - assert "Unknown LLM finding ID" in capsys.readouterr().err - rows[1]["llm_finding_id"] = programmatic_id - write_review(review, rows) - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - assert "Unknown LLM finding ID" in capsys.readouterr().err - rows[0]["llm_finding_id"] = "" - rows[0]["programmatic_finding_id"] = "" - rows[0]["match_reason"] = "" - rows[0]["classification"] = "ambiguous" - rows[1]["llm_finding_id"] = "" - rows[1]["match_reason"] = "" - rows[1]["classification"] = "ambiguous" - rows[2]["classification"] = "unavailable_evidence" - write_review(review, rows) - assert main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) == 0 - assert ( - "No Human findings were expected to be detected by the LLM" in (run / "report.md").read_text() - ) - assert "No Human findings were expected to be detected by the Programmatic checks" in (run / "report.md").read_text() - assert "Of all 3 Human findings, either part of the tool caught 0 (0.0%)" in (run / "report.md").read_text() - - -@pytest.mark.parametrize("model", ["claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5"]) -def test_live_execution_needs_explicit_approval( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, model: str, -) -> None: - """A noninteractive live flag without approval makes no paid request.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "available") - monkeypatch.setattr( - llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call") - ) - - class DeclinedInput: - def isatty(self) -> bool: - """Represent an unattended input stream.""" - return False - - monkeypatch.setattr("sys.stdin", DeclinedInput()) - assert main(args(human_audit, html, "--model", model, "--live")) == 0 - run = directory(tmp_path) - manifest = json.loads((run / "run.json").read_text()) - assert manifest["complete"] is False - assert manifest["incomplete_reason"] == "preview" - assert {path.name for path in run.iterdir()} == { - "review.csv", "snapshot.html", "normalized-programmatic-findings.json", - "run.json", "artifacts", - } - assert (run / "artifacts" / "prompts").is_dir() - assert not (run / "artifacts" / "raw-llm-responses.json").exists() - - -@pytest.mark.parametrize("failure", [True, False]) -@pytest.mark.parametrize("model", ["claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5"]) -def test_failed_or_over_budget_runs_keep_usage_and_cannot_report( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool, model: str, -) -> None: - """Failure and budget exhaustion stop after one request and remain inspectable.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - calls = 0 - - def call(self, prompt: str) -> dict: - """Return either a failure or an expensive successful response.""" - self.calls += 1 - return { - "success": not failure, - "response": json.dumps([{"problem": "Unclear page title"}]) if not failure else None, - "error": "synthetic failure" if failure else None, - "usage": {"input_tokens": 100000, "output_tokens": 20}, - } - - fake = FakeClient() - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: fake) - assert main(args(human_audit, html, "--model", model, "--live", "--approve-live")) == 1 - run = directory(tmp_path) - manifest = json.loads((run / "run.json").read_text()) - assert fake.calls == 1 - assert manifest["complete"] is False - assert manifest["incomplete_reason"] == ( - "synthetic failure" if failure else "cost guardrail exhausted" - ) - assert {path.name for path in run.iterdir()} == { - "review.csv", "snapshot.html", "normalized-llm-findings.json", - "normalized-programmatic-findings.json", "run.json", "artifacts", - } - findings = json.loads((run / "normalized-llm-findings.json").read_text()) - assert len(findings) == (0 if failure else 1) - assert all(item["run_id"] == run.name for item in findings) - assert manifest["usage"]["input_tokens"] == 100000 - assert manifest["configuration"]["model"] == model - assert all(item["model"] == model for item in findings) - assert float(manifest["estimated_cost_usd"]) > 0 - assert len(json.loads((run / "artifacts" / "raw-llm-responses.json").read_text())) == 1 - with pytest.raises(SystemExit): - main(["report", "--run-dir", str(run), "--reviewer", "Codex agent (operator-directed)"]) - - -@pytest.mark.parametrize("filename", ["human-findings.json", "pricing.json"]) -def test_report_requires_supporting_artifacts_without_reorganizing_old_files( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], filename: str, -) -> None: - """Root-level evidence is neither a fallback nor an automatic migration.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - def call(self, prompt: str) -> dict: - return {"success": True, "response": "[]", "usage": {}} - - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) - assert main(args(human_audit, html, "--live", "--approve-live")) == 0 - run = directory(tmp_path) - rows = read_review(run / "review.csv") - for row in rows: - row.update(classification="llm_eligible") - write_review(run / "review.csv", rows) - (run / "artifacts" / filename).rename(run / filename) - before = {path.relative_to(run): path.read_bytes() for path in run.rglob("*") if path.is_file()} - with pytest.raises(SystemExit) as error: - main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) - assert error.value.code == 2 - assert str(run / "artifacts" / filename) in capsys.readouterr().err - assert {path.relative_to(run): path.read_bytes() for path in run.rglob("*") if path.is_file()} == before - - -@pytest.mark.parametrize("filename,field", [ - ("run.json", "run_id"), - ("artifacts/human-findings.json", "human_audit_sha256"), - ("normalized-llm-findings.json", "run_id"), - ("normalized-llm-findings.json", "source"), - ("normalized-llm-findings.json", "page_url"), - ("normalized-programmatic-findings.json", "run_id"), - ("normalized-programmatic-findings.json", "source"), - ("normalized-programmatic-findings.json", "page_url"), -]) -def test_report_rejects_evidence_from_another_run( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], filename: str, field: str, -) -> None: - """Moving supporting evidence preserves all provenance validation.""" - human_audit, html = inputs(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") - - class FakeClient: - def call(self, prompt: str) -> dict: - return {"success": True, "response": '[{"problem": "Unclear link"}]', "usage": {}} - - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: FakeClient()) - assert main(args(human_audit, html, "--live", "--approve-live")) == 0 - run = directory(tmp_path) - path = run / filename - evidence = json.loads(path.read_text()) - record = evidence[0] if isinstance(evidence, list) else evidence - record[field] = "unrelated" - path.write_text(json.dumps(evidence)) - with pytest.raises(SystemExit) as error: - main(["report", "--run-dir", str(run), "--reviewer", "Test reviewer"]) - assert error.value.code == 2 - message = capsys.readouterr().err - if field == "human_audit_sha256": - assert "identity differs" in message - else: - assert "do not belong" in message or "does not belong" in message - assert not (run / "report.md").exists() diff --git a/tests/test_normalization.py b/tests/test_normalization.py deleted file mode 100644 index 0d4e0dd..0000000 --- a/tests/test_normalization.py +++ /dev/null @@ -1,133 +0,0 @@ -import json - -import pytest - -from processing_scripts.llm.registry import PROMPT_REGISTRY -from vision_aid.evaluation.normalization import ( - normalize_programmatic_findings, - normalize_prompt_response, -) - - -@pytest.mark.parametrize( - "prompt_name,output_type", - [ - (spec.name, spec.output_type) - for spec in PROMPT_REGISTRY - if not spec.is_summary - ], -) -def test_every_active_prompt_has_a_lossless_normalizer( - prompt_name: str, - output_type: str, -) -> None: - """Every active prompt can emit a canonical evidence-preserving finding.""" - item = { - "problem": "Synthetic accessibility failure", - "location": "hero image", - "element": "img.hero", - "wcag": ["1.1.1"], - "recommended_fix": "Synthetic fix retained as evidence", - } - response = item if output_type == "object" else [item] - - result = normalize_prompt_response( - prompt_name, - json.dumps(response), - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - - assert result.parse_status == "parsed" - assert len(result.findings) == 1 - finding = result.findings[0] - assert finding.prompt == prompt_name - assert finding.raw_source == item - assert finding.problem == "Synthetic accessibility failure" - assert finding.location == "hero image" - assert finding.wcag_evidence == ("1.1.1",) - - -def test_normalization_records_nonstandard_responses() -> None: - """All structured-output outcomes remain explicit and deterministic.""" - fenced = normalize_prompt_response( - "link_clarity", - '```json\n[{"issue": "Unclear link", "text": "Read more"}]\n```', - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - empty = normalize_prompt_response( - "link_clarity", - "[]", - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - malformed = normalize_prompt_response( - "link_clarity", - "successful but not JSON", - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - unknown = normalize_prompt_response( - "invented_prompt", - "[]", - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - - assert fenced.parse_status == "parsed" - assert empty.parse_status == "empty" - assert malformed.parse_status == "malformed" - assert malformed.findings == () - assert malformed.raw_response == "successful but not JSON" - assert unknown.parse_status == "unknown_prompt" - - -def test_programmatic_normalization_retains_rule_and_raw_source() -> None: - """Programmatic records keep their checker identity and raw evidence.""" - raw = { - "rule_id": "HEAD_002", - "message": "Multiple H1 elements", - "element": "

Second

", - "location": "line 20", - "wcag": "1.3.1", - } - - findings = normalize_programmatic_findings( - [raw], - run_id="run-1", - page_url="https://example.test/", - ) - - assert findings[0].source == "programmatic" - assert findings[0].problem_family == "HEAD_002" - assert findings[0].raw_source == raw - - -def test_normalization_emits_only_failures_from_assessment_responses() -> None: - """Compliant assessments stay observable without becoming findings.""" - clear_link = normalize_prompt_response( - "link_clarity", - '[{"text":"Donate","is_clear":true,"issues":[]}]', - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - table_failure = normalize_prompt_response( - "table_semantics", - '{"is_data_table":true,"caption_clear":false,' - '"issues":["Vague caption"]}', - run_id="run-1", - model="gpt-5.6-luna", - page_url="https://example.test/", - ) - - assert clear_link.parse_status == "empty" - assert clear_link.findings == () - assert table_failure.parse_status == "parsed" - assert table_failure.findings[0].problem == "Vague caption" From ce345d1ace35b4b67a71928870590fce4a38c936 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 17:09:08 -0400 Subject: [PATCH 061/126] tests: cover declined approval and excluded-row claims after review Test interactive and unattended approval refusal, reject coverage claimed on Not testable rows, pin the 18-category count, read report and snapshot text as UTF-8, and document nested helpers. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 10 ++++------ tests/test_evaluation.py | 42 ++++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index bf8f621..31e66dd 100644 --- a/README.md +++ b/README.md @@ -489,12 +489,10 @@ everything it does is free: - `index.html`'s inline JavaScript parses - the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit -Default pytest discovery is limited to `tests/`. There, `tests/test_evaluation.py` -covers the evaluator end to end through its CLI: preview, live runs against a -fake provider, review validation, scoring, and evidence checks. Detailed -evaluator cases can also be kept in the ignored `.local-tests/evaluation/` -folder and run separately with `uv run pytest -q .local-tests/evaluation`. -That folder is optional, so a fresh checkout and CI never depend on it. +Default pytest discovery is limited to `tests/`, where `tests/test_evaluation.py` +covers the evaluator end to end through its CLI with a fake provider. Optional +detailed evaluator cases live in the ignored `.local-tests/evaluation/` folder; +run them with `uv run pytest -q .local-tests/evaluation`. CI never needs them. ## Running the Pipeline diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index cf6f88d..b18c3da 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -16,6 +16,8 @@ MODELS = ("claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5") CATEGORIES = [spec.name for spec in PROMPT_REGISTRY if not spec.is_summary] +RUN_ROOT = {"review.csv", "snapshot.html", "normalized-programmatic-findings.json", + "run.json", "artifacts"} REVIEWER = "Test reviewer" # Synthetic markup that activates every non-summary prompt category. HTML = ( @@ -73,7 +75,7 @@ def run_cli(workspace: Path, *extra: str) -> int: def report(run: Path) -> str: """Score a reviewed run and return the Markdown report.""" assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 - return (run / "report.md").read_text() + return (run / "report.md").read_text(encoding="utf-8") def refused(run: Path, capsys: pytest.CaptureFixture[str], reviewer: str = REVIEWER) -> str: @@ -116,6 +118,7 @@ def fake_client(monkeypatch: pytest.MonkeyPatch, *responses: dict) -> list[str]: "usage": {"input_tokens": 1, "output_tokens": 1}} def call(prompt: str) -> dict: + """Return the next scripted response.""" prompts.append(prompt) return responses[len(prompts) - 1] if len(prompts) <= len(responses) else default @@ -124,17 +127,13 @@ def call(prompt: str) -> dict: return prompts -ROOT = {"review.csv", "snapshot.html", "normalized-programmatic-findings.json", - "run.json", "artifacts"} - - @pytest.mark.parametrize("model", [None, "claude-opus-5-5", "claude-sonnet-5-5"]) def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, model): """Credentials alone never buy requests, and all homepage rows survive.""" monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) assert run_cli(workspace, *(["--model", model] if model else [])) == 0 run = only_run(workspace) - assert {path.name for path in run.iterdir()} == ROOT + assert {path.name for path in run.iterdir()} == RUN_ROOT assert {path.name for path in (run / "artifacts").iterdir()} == { "human-findings.json", "pricing.json", "raw-programmatic-findings.json", "prompts", "payloads", @@ -142,8 +141,9 @@ def test_preview_preserves_evidence_without_provider_access(workspace, monkeypat manifest = load(run, "run.json") assert manifest["configuration"]["model"] == (model or MODELS[0]) assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" + assert len(CATEGORIES) == 18 assert [prompt["name"] for prompt in manifest["prompts"]] == CATEGORIES - assert (run / "snapshot.html").read_text() == HTML + assert (run / "snapshot.html").read_text(encoding="utf-8", errors="replace") == HTML audit = workspace / "human-audit.xlsx" assert manifest["human_audit_sha256"] == hashlib.sha256(audit.read_bytes()).hexdigest() assert manifest["snapshot_sha256"] == hashlib.sha256(HTML.encode()).hexdigest() @@ -167,11 +167,15 @@ def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( run_cli(workspace, "--model", "gpt-4o", "--live", "--approve-live") assert error.value.code == 2 and "invalid choice" in capsys.readouterr().err assert not (workspace / ".model-evaluation").exists() - monkeypatch.setattr("sys.stdin", SimpleNamespace(isatty=lambda: False)) - assert run_cli(workspace, "--live") == 0 - manifest = load(only_run(workspace), "run.json") - assert manifest["incomplete_reason"] == "preview" - assert not (only_run(workspace) / "artifacts" / "raw-llm-responses.json").exists() + monkeypatch.setattr("builtins.input", lambda _: "no") + for interactive in (False, True): + monkeypatch.setattr("sys.stdin", SimpleNamespace(isatty=lambda: interactive)) + assert run_cli(workspace, "--live") == 0 + runs = list((workspace / ".model-evaluation" / "runs").iterdir()) + assert len(runs) == 2 + for run in runs: + assert load(run, "run.json")["incomplete_reason"] == "preview" + assert not (run / "artifacts" / "raw-llm-responses.json").exists() @pytest.mark.parametrize("model,thinking,effort,cost", [ @@ -186,6 +190,7 @@ def test_presets_stream_every_category_and_report_saved_prices( requests, options = [], [] def create(**kwargs): + """Stream thinking, then final text, with cache usage.""" requests.append(kwargs) usage = SimpleNamespace(input_tokens=1000, output_tokens=1, cache_read_input_tokens=500, cache_creation_input_tokens=100) @@ -202,6 +207,7 @@ def create(**kwargs): ])) def client(**kwargs): + """Record client options and expose the fake Messages API.""" options.append(kwargs) return SimpleNamespace(messages=SimpleNamespace(create=create)) @@ -254,7 +260,8 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): {"classification": "programmatic", "programmatic_finding_id": programmatic, "match_reason": "HTML check"}, {"classification": "programmatic", "llm_finding_id": llm[1], "match_reason": "Cross-catch"}, - {"classification": "unavailable_evidence", "classification_reason": "Needs a browser"}, + {"classification": "unavailable_evidence", "classification_reason": "Needs a browser", + "llm_finding_id": "", "match_reason": ""}, {"classification": "ambiguous", "classification_reason": "Needs judgment"}, {"classification": "llm_eligible", "review_notes": "No full finding"}, ] @@ -264,6 +271,8 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): assert "needs a match reason" in refused(run, capsys) edit_review(run, {"llm_finding_id": programmatic, "match_reason": "Full issue"}) assert "Unknown LLM finding ID" in refused(run, capsys) + edit_review(run, *decisions[:3], {"llm_finding_id": llm[2], "match_reason": "Claimed"}) + assert "cannot claim coverage" in refused(run, capsys) edit_review(run, *decisions) text = report(run) assert ("Of the 2 Human findings the LLM could be expected to detect from the " @@ -295,10 +304,9 @@ def test_failure_or_exhaustion_stops_and_keeps_partial_evidence( workspace, monkeypatch, capsys, failure, calls, cost, reason, ): """Requests stop, billed usage and findings remain, and reporting refuses.""" - success = {"success": True, "response": '[{"problem": "Unclear title"}]', - "usage": {"input_tokens": 100000, "output_tokens": 20}} - stopped = {"success": False, "response": None, "error": "synthetic failure", - "usage": {"input_tokens": 100000, "output_tokens": 20}} + usage = {"input_tokens": 100000, "output_tokens": 20} + success = {"success": True, "response": '[{"problem": "Unclear title"}]', "usage": usage} + stopped = {"success": False, "response": None, "error": "synthetic failure", "usage": usage} prompts = fake_client(monkeypatch, success, stopped if failure else success) limit = "1" if failure else "0.01" assert run_cli(workspace, "--max-cost-usd", limit, "--live", "--approve-live") == 1 From b93586d3231df39f623726b8fafdcbbf7e2e326d Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 17:32:55 -0400 Subject: [PATCH 062/126] fix(evaluation): parse LLM responses exactly as the API does The evaluator's own fence stripper required the code fence to be the whole response, so a valid JSON block followed by a prose note was flagged as unreadable and its findings dropped. The API's safe_parse_json reads the first fenced block and ignores the rest, so the evaluator undercounted what the API would report. Normalization now calls safe_parse_json directly, so an Unreadable response is exactly one the API would skip. The report sentence and glossary use that term. Co-Authored-By: Claude Opus 5.5 (1M context) --- CONTEXT.md | 4 ++++ tests/test_evaluation.py | 11 +++++++++-- vision_aid/evaluation/normalization.py | 12 ++++-------- vision_aid/evaluation/review.py | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index c0e8d79..fb21b5d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -32,6 +32,10 @@ _Avoid_: Deterministic checker. An accessibility problem reported by the LLM, kept with the response it came from. _Avoid_: Audit finding, Human finding, Programmatic finding. +**Unreadable response**: +An LLM response the tool itself cannot parse and would therefore skip. A response counts as readable whenever the tool could use it, even if it contains extra text or needed repair. +_Avoid_: Malformed output, format failure (in reports). + **Programmatic finding**: An accessibility problem reported by the Programmatic checks. _Avoid_: Audit finding, LLM finding. diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index b18c3da..9dac440 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -246,12 +246,18 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): monkeypatch, {"success": True, "response": "not JSON", "usage": {}}, {"success": True, "response": "[]", "usage": {}}, + # The API reads the first fenced block and ignores trailing prose. + {"success": True, "usage": {}, "response": ( + '```json\n[{"problem": "Fenced with a note"}]\n```\n\n**Note:** Context was limited.' + )}, ) assert run_cli(workspace, "--live", "--approve-live") == 0 run = only_run(workspace) manifest = load(run, "run.json") assert manifest["complete"] is True and manifest["format_failures"] == ["page_title"] - llm = [item["finding_id"] for item in load(run, "normalized-llm-findings.json")] + findings = load(run, "normalized-llm-findings.json") + assert findings[0]["problem"] == "Fenced with a note" + llm = [item["finding_id"] for item in findings] assert len(llm) == len(CATEGORIES) - 2 programmatic = load(run, "normalized-programmatic-findings.json")[0]["finding_id"] assert "classification" in refused(run, capsys) @@ -286,7 +292,8 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): excluded = text.split("## Not testable from the page file\n", 1)[1] assert "### Hidden menu" in excluded and "Needs a browser" in excluded assert "### Unclear claim" in excluded and "Needs judgment" in excluded - assert "The LLM returned unreadable output for 1 of 18 checks: page title check." in text + assert ("1 of 18 LLM checks returned an Unreadable response that the tool " + "would skip: page title check.") in text assert all(finding_id not in text for finding_id in [*llm, programmatic]) edit_review(run, *[{"classification": "ambiguous", "llm_finding_id": "", "programmatic_finding_id": "", "match_reason": ""}] * 6) diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index 88a170a..61e56c6 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from typing import Any +from entry_points.generate_report import safe_parse_json from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec from vision_aid.evaluation.schemas import CanonicalFinding @@ -55,12 +56,6 @@ class NormalizationResult: error: str | None = None -def _strip_fence(value: str) -> str: - """Remove one optional Markdown JSON fence.""" - match = re.fullmatch(r"\s*```(?:json)?\s*(.*?)\s*```\s*", value, re.DOTALL) - return match.group(1) if match else value.strip() - - def _first_text(item: dict[str, Any], keys: Iterable[str]) -> str: """Return the first populated evidence field as stable text.""" for key in keys: @@ -194,8 +189,9 @@ def normalize_prompt_response( error=(f"No fixed evaluation normalizer exists for {prompt_name!r}."), ) try: - parsed = json.loads(_strip_fence(raw_response)) - except (json.JSONDecodeError, TypeError) as error: + # Parse exactly as the API's report does, so only responses it skips are unreadable. + parsed = safe_parse_json(raw_response) + except (json.JSONDecodeError, TypeError, ValueError) as error: return NormalizationResult( prompt=prompt_name, parse_status="malformed", diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 7a4f92e..546c14c 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -193,7 +193,7 @@ def write_report( ]) if manifest["format_failures"]: checks = ", ".join(_md(name.replace("_", " ") + " check") for name in manifest["format_failures"]) - lines.extend([f"The LLM returned unreadable output for {len(manifest['format_failures'])} of {len(manifest['prompts'])} checks: {checks}.", ""]) + lines.extend([f"{len(manifest['format_failures'])} of {len(manifest['prompts'])} LLM checks returned an Unreadable response that the tool would skip: {checks}.", ""]) lines.extend([ "## Human findings", "", "| Issue Title | WCAG | Expected to be caught by | LLM | Programmatic checks |", From ab42455945b1ba5436dbb99202ae1c69563b109a Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 17:34:02 -0400 Subject: [PATCH 063/126] fix(evaluation): catch exactly the parse errors the API skips Co-Authored-By: Claude Opus 5.5 (1M context) --- vision_aid/evaluation/normalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index 61e56c6..58535df 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -191,7 +191,7 @@ def normalize_prompt_response( try: # Parse exactly as the API's report does, so only responses it skips are unreadable. parsed = safe_parse_json(raw_response) - except (json.JSONDecodeError, TypeError, ValueError) as error: + except (json.JSONDecodeError, ValueError) as error: return NormalizationResult( prompt=prompt_name, parse_status="malformed", From 77c791b877dac1eb0306538985d3f4301e3157bd Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 18:09:12 -0400 Subject: [PATCH 064/126] fix(evaluation): count LLM findings with the tool's per-check rules Read each parsed response with the tool's report normalizers, imported from generate_report, one row per LLM finding, before its false-positive filter and deduplication. Six checks without a tool normalizer keep an evaluator rule that needs a listed problem, and their findings are marked screen_only and labelled in report.md. Responses a rule cannot read are recorded as Unreadable without ending the run. Co-Authored-By: Claude Opus 5.5 (1M context) --- CONTEXT.md | 4 +- README.md | 51 ++++++- tests/test_evaluation.py | 103 +++++++++++++- vision_aid/evaluation/normalization.py | 190 +++++++++++-------------- vision_aid/evaluation/review.py | 6 +- vision_aid/evaluation/schemas.py | 2 + 6 files changed, 241 insertions(+), 115 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index fb21b5d..dcb9360 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -29,11 +29,11 @@ The part of the tool that applies fixed rules to the page's HTML. It uses no AI _Avoid_: Deterministic checker. **LLM finding**: -An accessibility problem reported by the LLM, kept with the response it came from. +One accessibility problem the tool reads from an LLM response, before any false-positive filtering or deduplication, kept with the response it came from. _Avoid_: Audit finding, Human finding, Programmatic finding. **Unreadable response**: -An LLM response the tool itself cannot parse and would therefore skip. A response counts as readable whenever the tool could use it, even if it contains extra text or needed repair. +An LLM response the tool itself cannot parse or interpret, so it yields no LLM findings. A response counts as readable whenever the tool could use it, even if it contains extra text or needed repair. _Avoid_: Malformed output, format failure (in reports). **Programmatic finding**: diff --git a/README.md b/README.md index 31e66dd..6415001 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,55 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report Programmatic rates but remain in the Overall denominator. Reporting rejects incomplete runs, missing classifications, and invalid or reused finding IDs. +### How the evaluator reads LLM responses + +The evaluator measures the tool as it is, so it reads each LLM response the +way the tool's CSV report does, using the tool's own code rather than a copy. + +**Parsing.** Each response is parsed with the tool's JSON parser, including its +code-fence and repair steps. A response the tool cannot parse, or whose shape +the check's rule cannot read (for example one object where a list is expected), +is an Unreadable response: it gives no LLM findings, `run.json` lists the check +under `format_failures`, and the run continues with the next check. + +**What counts as a finding.** For the 12 checks the tool's CSV report covers, +the tool's own per-check rule decides, and each row it would write becomes one +LLM finding: + +- Page title and landmarks: one finding per entry in `issues`. Yes/no answers + such as `is_descriptive` are ignored. +- Headings: one finding per entry in `issues`, plus one per entry in + `vague_headings`. +- Link clarity, iframe titles, label quality, and decorative verification: one + finding per item whose verdict (`is_clear`, `is_descriptive`, or + `likely_decorative`) is `false`. A missing verdict counts as a pass. +- Required field indicators, informative alt quality, actionable image alt, + SVGs, and icon fonts: one finding per item with a non-empty `issues` list, + with its issues joined. An item the model judged acceptable gives no + finding, even if a yes/no answer such as `has_accessible_name` is `false`. + +Each finding's problem, element, and location are the text the tool would +write in its row. `raw_source` keeps the full item, or for page title, +headings, and landmarks the whole response plus the specific issue. + +**What is skipped.** The tool's false-positive filter and LLM deduplication are +not applied. The filter guesses from the page HTML and can remove a finding +other than the one it matched; deduplication costs extra LLM calls and is not +repeatable. Evaluation counts therefore measure what the model reported and +will not match the rows in the tool's downloadable CSV report. + +**Checks shown on screen only.** Six checks have no rule in the tool's CSV +report; their results appear only in the web app's on-screen panel: +`table_semantics`, `placeholder_as_label`, `group_labels`, `form_instructions`, +`complex_descriptions`, and `media_captions`. The evaluator still reads them, +giving at most one finding per item, and only when the item's `issues` list is +non-empty (for tables, `issues` or `header_clarity_issues`). Every +`placeholder_as_label` item is a finding, because that check only receives +placeholder-only fields. Yes/no answers never create a finding on their own. +A table response may be one object or a list. These findings have +`"screen_only": true`, and `report.md` labels them "shown on screen only, not +in the tool's CSV report". + ### Evaluation run files The run root holds the review evidence and scored results. Supporting files @@ -353,7 +402,7 @@ compatibility fallback or automatic migration. | --- | --- | | `review.csv` | Original Human audit evidence, source sheet and row, stable IDs, and the decision columns the reviewer edits. | | `snapshot.html` | The full Benchmark snapshot used to judge testability and coverage. | -| `normalized-llm-findings.json` | Parsed LLM findings with `finding_id` values for `review.csv`. A malformed reply can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | +| `normalized-llm-findings.json` | LLM findings with `finding_id` values for `review.csv`, read as described above. An Unreadable response can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | | `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | | `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | | `report.md` | Scored results generated after successful reporting. | diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 9dac440..fb52dc5 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -114,7 +114,7 @@ def edit_review(run: Path, *decisions: dict[str, str]) -> None: def fake_client(monkeypatch: pytest.MonkeyPatch, *responses: dict) -> list[str]: """Replace the request client, replaying responses and recording prompts.""" prompts: list[str] = [] - default = {"success": True, "response": '[{"problem": "Vague link"}]', + default = {"success": True, "response": '[{"issues": ["Vague link"]}]', "usage": {"input_tokens": 1, "output_tokens": 1}} def call(prompt: str) -> dict: @@ -200,7 +200,7 @@ def create(**kwargs): SimpleNamespace(type="content_block_start", content_block=SimpleNamespace(type="thinking", thinking="SECRET")), SimpleNamespace(type="content_block_start", content_block=SimpleNamespace( - type="text", text='[{"problem": "Synthetic failure", "location": "body"}]')), + type="text", text='[{"issues": ["Synthetic failure"], "location_hint": "body"}]')), SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), usage=SimpleNamespace(output_tokens=200)), SimpleNamespace(type="message_stop"), @@ -227,8 +227,16 @@ def client(**kwargs): raw = load(run, "artifacts/raw-llm-responses.json") assert all(item["result"]["provider"]["response_model"] == model for item in raw) assert "SECRET" not in json.dumps(raw) + assert manifest["format_failures"] == [ + "page_title", "heading_structure", "landmark_structure", + ] findings = load(run, "normalized-llm-findings.json") - assert [finding["prompt"] for finding in findings] == CATEGORIES + assert [finding["prompt"] for finding in findings] == [ + "table_semantics", "placeholder_as_label", "group_labels", + "required_field_indicators", "form_instructions", "informative_alt_quality", + "actionable_image_alt", "complex_descriptions", "svg_accessibility", + "icon_font_accessibility", "media_captions", + ] assert all(finding["model"] == model for finding in findings) assert all(finding["problem"] == "Synthetic failure" for finding in findings) assert all(finding["finding_id"].startswith("llm-") for finding in findings) @@ -245,11 +253,15 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): fake_client( monkeypatch, {"success": True, "response": "not JSON", "usage": {}}, - {"success": True, "response": "[]", "usage": {}}, + {"success": True, "response": '{"issues": []}', "usage": {}}, # The API reads the first fenced block and ignores trailing prose. {"success": True, "usage": {}, "response": ( - '```json\n[{"problem": "Fenced with a note"}]\n```\n\n**Note:** Context was limited.' + '```json\n[{"text": "Click here", "is_clear": false, "reason": "Fenced with a note"}]' + '\n```\n\n**Note:** Context was limited.' )}, + {"success": True, "response": "[]", "usage": {}}, + {"success": True, "response": "[]", "usage": {}}, + {"success": True, "response": '{"issues": []}', "usage": {}}, ) assert run_cli(workspace, "--live", "--approve-live") == 0 run = only_run(workspace) @@ -258,7 +270,7 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): findings = load(run, "normalized-llm-findings.json") assert findings[0]["problem"] == "Fenced with a note" llm = [item["finding_id"] for item in findings] - assert len(llm) == len(CATEGORIES) - 2 + assert len(llm) == 11 programmatic = load(run, "normalized-programmatic-findings.json")[0]["finding_id"] assert "classification" in refused(run, capsys) decisions = [ @@ -312,7 +324,7 @@ def test_failure_or_exhaustion_stops_and_keeps_partial_evidence( ): """Requests stop, billed usage and findings remain, and reporting refuses.""" usage = {"input_tokens": 100000, "output_tokens": 20} - success = {"success": True, "response": '[{"problem": "Unclear title"}]', "usage": usage} + success = {"success": True, "response": '{"issues": ["Unclear title"]}', "usage": usage} stopped = {"success": False, "response": None, "error": "synthetic failure", "usage": usage} prompts = fake_client(monkeypatch, success, stopped if failure else success) limit = "1" if failure else "0.01" @@ -351,3 +363,80 @@ def test_report_rejects_unrelated_or_missing_evidence( (evidence[0] if isinstance(evidence, list) else evidence)[field] = "unrelated" (run / name).write_text(json.dumps(evidence)) assert message in refused(run, capsys) + + +def test_llm_findings_follow_the_tools_rules(workspace, monkeypatch): + """Findings come from the tool's per-check rules, before filtering or deduplication.""" + def ok(response: object) -> dict: + """Script one successful response as JSON text.""" + return {"success": True, "response": json.dumps(response), "usage": {}} + + fake_client( + monkeypatch, + ok({"is_descriptive": False, "issues": ["Title is generic"]}), + ok({"structure_clear": False, "issues": ["Skips h2", "Two h1s", "No outline"], + "vague_headings": ["News", "More"]}), + # Kept although the tool's false-positive filter would drop it. + ok([{"text": "Contact", "is_clear": False, "reason": "Ambiguous", + "location_hint": "Footer"}, {"text": "Home", "is_clear": True}]), + # The table prompt asks for one object, not a list. + ok({"is_data_table": True, "caption_clear": False, + "header_clarity_issues": [], "issues": ["No caption"]}), + # A list check answered with one object is unreadable. + ok({"title": "Video", "is_descriptive": False}), + ok({"structure_appropriate": False, "issues": []}), + ok([]), + ok([{"field_id": None, "placeholder": "Email", "reason": "Placeholder only"}]), + ok([{"legend_is_meaningful": False, "issues": []}]), + ok([{"field_id": "n", "requirement_in_label": False, + "requirement_in_instructions": False, "issues": []}]), + ok([]), + ok([{"src": "team.png", "alt": "Team", "quality": "poor", + "issues": ["Too short", "Omits names"]}]), + ok([]), + ok([]), + ok([]), + ok([{"has_accessible_name": False, "issues": [], + "reason": "No 1.1.1 failure; presentational"}]), + ok([]), + ok([{"src": "v.mp4", "has_captions_track": False, "issues": []}, + {"src": "v.mp4", "has_captions_track": False, "issues": ["No captions track"]}]), + ) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["format_failures"] == ["iframe_titles"] + findings = load(run, "normalized-llm-findings.json") + assert [(item["prompt"], item["problem"]) for item in findings] == [ + ("page_title", "Title is generic"), + ("heading_structure", "Skips h2"), + ("heading_structure", "Two h1s"), + ("heading_structure", "No outline"), + ("heading_structure", 'Heading "News" is vague or unclear'), + ("heading_structure", 'Heading "More" is vague or unclear'), + ("link_clarity", "Ambiguous"), + ("table_semantics", "No caption"), + ("placeholder_as_label", "Placeholder only"), + ("informative_alt_quality", "Too short; Omits names"), + ("media_captions", "No captions track"), + ] + assert len({item["finding_id"] for item in findings}) == len(findings) + screen_only = [item["prompt"] for item in findings if item["screen_only"]] + assert screen_only == ["table_semantics", "placeholder_as_label", "media_captions"] + link = findings[6] + assert (link["element"], link["location"]) == (' "Contact"', "Footer") + assert link["raw_source"]["text"] == "Contact" and link["wcag_evidence"] == ["2.4.4"] + vague = findings[4]["raw_source"] + assert vague["issue"] == "News" and vague["response"]["vague_headings"] == ["News", "More"] + media = findings[-1] + assert media["raw_source"]["issues"] == ["No captions track"] + edit_review( + run, *[{"classification": "llm_eligible"}] * 5, + {"classification": "llm_eligible", "llm_finding_id": media["finding_id"], + "match_reason": "No captions on the video"}, + ) + text = report(run) + assert ("LLM findings are read from each response with the tool's own rules, " + "before its false-positive filter and deduplication.") in text + assert ("- LLM (media captions check; shown on screen only, not in the tool's " + "CSV report): No captions track") in text diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index 58535df..769fae7 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from typing import Any -from entry_points.generate_report import safe_parse_json +from entry_points.generate_report import NORMALIZERS, safe_parse_json from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec from vision_aid.evaluation.schemas import CanonicalFinding @@ -24,14 +24,6 @@ "actual_result", "summary", ) -_EXPLICIT_PROBLEM_KEYS = ( - "problem", - "issue", - "issue_title", - "description", - "finding", - "actual_result", -) _LOCATION_KEYS = ( "location", "location_hint", @@ -43,6 +35,8 @@ ) _ELEMENT_KEYS = ("element", "element_name", "tag", "html", "selector") _WCAG_KEYS = ("wcag", "wcag_sc", "wcag_criteria", "criterion", "criteria") +# Checks without a tool normalizer only reach users on screen; these fields name their element. +_SCREEN_ONLY_ELEMENT_KEYS = ("field_id", "src", "legend", "placeholder") @dataclass(frozen=True) @@ -81,93 +75,76 @@ def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: return tuple(dict.fromkeys(criteria or defaults)) -def _items(parsed: Any, spec: PromptSpec) -> list[dict[str, Any]] | None: - """Return source finding objects from either supported JSON shape.""" - if isinstance(parsed, list): - return [item for item in parsed if isinstance(item, dict)] - if not isinstance(parsed, dict): - return None - for key in ("findings", "violations", "results"): - nested = parsed.get(key) - if isinstance(nested, list) and all(isinstance(item, dict) for item in nested): - return [item for item in nested if isinstance(item, dict)] - return [parsed] if parsed else [] +def _listed(value: Any) -> list[Any]: + """Treat a missing or empty value as no issues and a lone value as one.""" + if not value: + return [] + return value if isinstance(value, list) else [value] -def _has_values(value: Any) -> bool: - """Treat non-empty strings, mappings, and sequences as evidence.""" - return value not in (None, "", [], {}, ()) +def _objects(parsed: Any, shape: str) -> list[dict[str, Any]]: + """Return a response's items, rejecting a shape the check's rule cannot read.""" + if not isinstance(parsed, list) or not all(isinstance(item, dict) for item in parsed): + raise ValueError(f"Expected a JSON {shape} response.") + return parsed -def _is_failure(prompt: str, item: dict[str, Any]) -> bool: - """Apply the fixed failure predicate for one active prompt category.""" - if any(_has_values(item.get(key)) for key in _EXPLICIT_PROBLEM_KEYS): - return True - if _has_values(item.get("issues")): - return True - if prompt == "page_title": - return item.get("is_descriptive") is False or item.get("matches_h1") is False - if prompt == "heading_structure": - return item.get("structure_clear") is False or _has_values( - item.get("vague_headings") - ) - if prompt == "link_clarity": - return item.get("is_clear") is False - if prompt == "table_semantics": - return item.get("caption_clear") is False or _has_values( - item.get("header_clarity_issues") - ) - if prompt == "iframe_titles": - return item.get("is_descriptive") is False - if prompt == "landmark_structure": - return item.get("structure_appropriate") is False - if prompt == "label_quality": - return item.get("is_descriptive") is False - if prompt == "placeholder_as_label": - return True - if prompt == "group_labels": - return item.get("legend_is_meaningful") is False - if prompt == "required_field_indicators": - return ( - item.get("requirement_in_label") is False - and item.get("requirement_in_instructions") is False - ) - if prompt == "form_instructions": - return item.get("instructions_are_helpful") is False - if prompt == "informative_alt_quality": - return item.get("quality") == "poor" - if prompt == "decorative_verification": - return item.get("likely_decorative") is False - if prompt == "actionable_image_alt": - return item.get("describes_action_not_appearance") is False - if prompt == "complex_descriptions": - return item.get("alt_is_sufficient") is False or ( - item.get("long_description_needed") is True - and item.get("long_description_adequate") is not True - ) - if prompt == "svg_accessibility": - return ( - item.get("has_accessible_name") is False - or item.get("title_is_meaningful") is False +def _tool_findings( + normalizer: Any, parsed: Any, spec: PromptSpec +) -> list[tuple[int, Any, str, str, str]]: + """Read findings with the tool's report rule, pairing each row with its source.""" + wcag = ", ".join(spec.wcag_criteria) + if spec.output_type == "object": + if not isinstance(parsed, dict): + raise ValueError("Expected a JSON object response.") + rows = normalizer(parsed, wcag=wcag) + # Page-level rules write one row per issue, then one per vague heading. + issues = [*(parsed.get("issues") or []), *(parsed.get("vague_headings") or [])] + sources = [{"response": parsed, "issue": issue} for issue in issues] + paired = list(enumerate(zip(sources, rows))) + else: + # One item at a time, so each row keeps the item it came from. + paired = [ + (index, (item, row)) + for index, item in enumerate(_objects(parsed, "array")) + for row in normalizer([item], wcag=wcag) + ] + return [ + ( + position, + source, + row.actual_result or row.issue_title, + row.element_name, + row.steps_to_reproduce, ) - if prompt == "icon_font_accessibility": - return item.get("pattern") in {"unlabeled_control", "missing_label"} - if prompt == "media_captions": - return ( - item.get("has_captions_track") is False or item.get("has_controls") is False - ) - return False + for position, (source, row) in paired + ] -def _problem(item: dict[str, Any], prompt_name: str) -> str: - """Choose a readable problem statement while retaining the raw object.""" - explicit = _first_text(item, _EXPLICIT_PROBLEM_KEYS) - if explicit: - return explicit - issues = item.get("issues") - if isinstance(issues, list) and issues: - return "; ".join(str(value) for value in issues) - return _first_text(item, ("reason", "summary")) or prompt_name +def _screen_only_findings( + prompt_name: str, parsed: Any +) -> list[tuple[int, Any, str, str, str]]: + """Read a check the tool's report skips: a finding needs a listed problem.""" + if prompt_name == "table_semantics" and isinstance(parsed, dict): + # The table prompt asks for one object rather than a list. + parsed = [parsed] + findings = [] + for index, item in enumerate(_objects(parsed, "array")): + issues = _listed(item.get("issues")) + if prompt_name == "table_semantics": + issues += _listed(item.get("header_clarity_issues")) + # Placeholder-only fields are the problem; that check lists no issues. + if not issues and prompt_name != "placeholder_as_label": + continue + problem = "; ".join(str(issue) for issue in issues) + findings.append(( + index, + item, + problem or str(item.get("reason") or "") or prompt_name, + _first_text(item, _SCREEN_ONLY_ELEMENT_KEYS), + str(item.get("location_hint") or ""), + )) + return findings def normalize_prompt_response( @@ -178,7 +155,7 @@ def normalize_prompt_response( model: str, page_url: str, ) -> NormalizationResult: - """Normalize one prompt directly, without suppression or deduplication.""" + """Read one response with the tool's rules, without its filter or deduplication.""" spec = _PROMPTS.get(prompt_name) if spec is None: return NormalizationResult( @@ -199,26 +176,30 @@ def normalize_prompt_response( findings=(), error=str(error), ) - items = _items(parsed, spec) - if items is None: + normalizer = NORMALIZERS.get(prompt_name) + try: + if normalizer is None: + sources = _screen_only_findings(prompt_name, parsed) + else: + sources = _tool_findings(normalizer, parsed, spec) + except Exception as error: # noqa: BLE001 + # A shape the check's rule cannot read is unreadable, not fatal to the run. return NormalizationResult( prompt=prompt_name, parse_status="malformed", raw_response=raw_response, findings=(), - error=f"Expected a JSON {spec.output_type} response.", + error=str(error) or type(error).__name__, ) findings = [] - for index, item in enumerate(items): - if not _is_failure(prompt_name, item): - continue - canonical_item = json.dumps( - item, + for position, source, problem, element, location in sources: + canonical_source = json.dumps( + source, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) - identity = f"{run_id}\0{prompt_name}\0{index}\0{canonical_item}".encode() + identity = f"{run_id}\0{prompt_name}\0{position}\0{canonical_source}".encode() findings.append( CanonicalFinding( finding_id="llm-" + hashlib.sha256(identity).hexdigest()[:20], @@ -229,12 +210,13 @@ def normalize_prompt_response( checklist=spec.checklist, page_url=page_url, problem_family=prompt_name, - problem=_problem(item, prompt_name), - element=_first_text(item, _ELEMENT_KEYS), - location=_first_text(item, _LOCATION_KEYS), - wcag_evidence=_wcag(item, spec.wcag_criteria), - raw_source=item, + problem=problem, + element=element, + location=location, + wcag_evidence=tuple(spec.wcag_criteria), + raw_source=source, parse_status="parsed", + screen_only=normalizer is None, ) ) return NormalizationResult( diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 546c14c..c2b90bb 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -170,6 +170,7 @@ def write_report( "", "The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.", "This API uses LLM checks and Programmatic checks. LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.", + "LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.", "The tool saw only saved HTML, not a live browser or screen reader. This evaluation covers the homepage only, including Global findings that apply there.", "A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.", "", @@ -230,8 +231,11 @@ def write_report( for kind in ("llm", "programmatic"): for identity in matches[kind]: finding = findings[identity] + check = finding["prompt"].replace("_", " ") + " check" + if finding.get("screen_only"): + check += "; shown on screen only, not in the tool's CSV report" label = ( - f"LLM ({finding['prompt'].replace('_', ' ')} check)" + f"LLM ({check})" if kind == "llm" else f"Programmatic check {finding['prompt']}" ) lines.extend([f"- {_md(label)}: {_md(finding['problem'])}", "", f" - Why this counts: {_md(decision['match_reason'])}"]) diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index facd746..f46de59 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -46,3 +46,5 @@ class CanonicalFinding: wcag_evidence: tuple[str, ...] raw_source: Any parse_status: str + # True when the tool shows this finding on screen but leaves it out of its CSV report. + screen_only: bool = False From 8a59f67c3e84ae2a4e49edb38ee8a1dbcd315be3 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 18:11:09 -0400 Subject: [PATCH 065/126] fix(evaluation): keep table evidence intact and reject text issue lists Copy listed-problem fields so table_semantics no longer appends header clarity issues into the saved source item. Treat text where issues, vague_headings, or header_clarity_issues should be a list as an Unreadable response instead of one finding per character. Name the finding tuple and helpers. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 5 ++- tests/test_evaluation.py | 11 +++-- vision_aid/evaluation/normalization.py | 59 ++++++++++++++++---------- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 6415001..6358712 100644 --- a/README.md +++ b/README.md @@ -343,8 +343,9 @@ way the tool's CSV report does, using the tool's own code rather than a copy. **Parsing.** Each response is parsed with the tool's JSON parser, including its code-fence and repair steps. A response the tool cannot parse, or whose shape -the check's rule cannot read (for example one object where a list is expected), -is an Unreadable response: it gives no LLM findings, `run.json` lists the check +the check's rule cannot read (for example one object where a list is expected, +or text where `issues`, `vague_headings`, or `header_clarity_issues` should be +a list), is an Unreadable response: it gives no LLM findings, `run.json` lists the check under `format_failures`, and the run continues with the next check. **What counts as a finding.** For the 12 checks the tool's CSV report covers, diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index fb52dc5..877552e 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -381,10 +381,11 @@ def ok(response: object) -> dict: "location_hint": "Footer"}, {"text": "Home", "is_clear": True}]), # The table prompt asks for one object, not a list. ok({"is_data_table": True, "caption_clear": False, - "header_clarity_issues": [], "issues": ["No caption"]}), + "header_clarity_issues": ["Year is vague"], "issues": ["No caption"]}), # A list check answered with one object is unreadable. ok({"title": "Video", "is_descriptive": False}), - ok({"structure_appropriate": False, "issues": []}), + # Text where a list belongs is unreadable, not one finding per character. + ok({"structure_appropriate": False, "issues": "No main"}), ok([]), ok([{"field_id": None, "placeholder": "Email", "reason": "Placeholder only"}]), ok([{"legend_is_meaningful": False, "issues": []}]), @@ -405,7 +406,8 @@ def ok(response: object) -> dict: assert run_cli(workspace, "--live", "--approve-live") == 0 run = only_run(workspace) manifest = load(run, "run.json") - assert manifest["complete"] is True and manifest["format_failures"] == ["iframe_titles"] + assert manifest["complete"] is True + assert manifest["format_failures"] == ["iframe_titles", "landmark_structure"] findings = load(run, "normalized-llm-findings.json") assert [(item["prompt"], item["problem"]) for item in findings] == [ ("page_title", "Title is generic"), @@ -415,7 +417,7 @@ def ok(response: object) -> dict: ("heading_structure", 'Heading "News" is vague or unclear'), ("heading_structure", 'Heading "More" is vague or unclear'), ("link_clarity", "Ambiguous"), - ("table_semantics", "No caption"), + ("table_semantics", "No caption; Year is vague"), ("placeholder_as_label", "Placeholder only"), ("informative_alt_quality", "Too short; Omits names"), ("media_captions", "No captions track"), @@ -426,6 +428,7 @@ def ok(response: object) -> dict: link = findings[6] assert (link["element"], link["location"]) == (' "Contact"', "Footer") assert link["raw_source"]["text"] == "Contact" and link["wcag_evidence"] == ["2.4.4"] + assert findings[7]["raw_source"]["issues"] == ["No caption"] vague = findings[4]["raw_source"] assert vague["issue"] == "News" and vague["response"]["vague_headings"] == ["News", "More"] media = findings[-1] diff --git a/vision_aid/evaluation/normalization.py b/vision_aid/evaluation/normalization.py index 769fae7..7ab336b 100644 --- a/vision_aid/evaluation/normalization.py +++ b/vision_aid/evaluation/normalization.py @@ -5,11 +5,11 @@ import hashlib import json import re -from collections.abc import Iterable, Sequence +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, NamedTuple -from entry_points.generate_report import NORMALIZERS, safe_parse_json +from entry_points.generate_report import NORMALIZERS, ReportRow, safe_parse_json from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec from vision_aid.evaluation.schemas import CanonicalFinding @@ -75,42 +75,57 @@ def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: return tuple(dict.fromkeys(criteria or defaults)) -def _listed(value: Any) -> list[Any]: - """Treat a missing or empty value as no issues and a lone value as one.""" - if not value: +class _Source(NamedTuple): + """One LLM finding's text, with the response part it came from.""" + + position: int + source: Any + problem: str + element: str + location: str + + +def _issue_list(item: dict[str, Any], key: str) -> list[Any]: + """Copy a listed-problems field, rejecting text that would split per character.""" + value = item.get(key) + if value is None: return [] - return value if isinstance(value, list) else [value] + if not isinstance(value, list): + raise TypeError(f"Expected {key!r} to be a JSON array.") + return list(value) -def _objects(parsed: Any, shape: str) -> list[dict[str, Any]]: - """Return a response's items, rejecting a shape the check's rule cannot read.""" +def _item_list(parsed: Any) -> list[dict[str, Any]]: + """Return an array response's items, rejecting a shape no rule can read.""" if not isinstance(parsed, list) or not all(isinstance(item, dict) for item in parsed): - raise ValueError(f"Expected a JSON {shape} response.") + raise ValueError("Expected a JSON array response.") + for item in parsed: + _issue_list(item, "issues") return parsed def _tool_findings( - normalizer: Any, parsed: Any, spec: PromptSpec -) -> list[tuple[int, Any, str, str, str]]: + normalizer: Callable[..., list[ReportRow]], parsed: Any, spec: PromptSpec +) -> list[_Source]: """Read findings with the tool's report rule, pairing each row with its source.""" wcag = ", ".join(spec.wcag_criteria) if spec.output_type == "object": if not isinstance(parsed, dict): raise ValueError("Expected a JSON object response.") - rows = normalizer(parsed, wcag=wcag) # Page-level rules write one row per issue, then one per vague heading. - issues = [*(parsed.get("issues") or []), *(parsed.get("vague_headings") or [])] + issues = _issue_list(parsed, "issues") + _issue_list(parsed, "vague_headings") + rows = normalizer(parsed, wcag=wcag) sources = [{"response": parsed, "issue": issue} for issue in issues] paired = list(enumerate(zip(sources, rows))) else: # One item at a time, so each row keeps the item it came from. paired = [ (index, (item, row)) - for index, item in enumerate(_objects(parsed, "array")) + for index, item in enumerate(_item_list(parsed)) for row in normalizer([item], wcag=wcag) ] return [ - ( + _Source( position, source, row.actual_result or row.issue_title, @@ -121,23 +136,21 @@ def _tool_findings( ] -def _screen_only_findings( - prompt_name: str, parsed: Any -) -> list[tuple[int, Any, str, str, str]]: +def _screen_only_findings(prompt_name: str, parsed: Any) -> list[_Source]: """Read a check the tool's report skips: a finding needs a listed problem.""" if prompt_name == "table_semantics" and isinstance(parsed, dict): # The table prompt asks for one object rather than a list. parsed = [parsed] findings = [] - for index, item in enumerate(_objects(parsed, "array")): - issues = _listed(item.get("issues")) + for index, item in enumerate(_item_list(parsed)): + issues = _issue_list(item, "issues") if prompt_name == "table_semantics": - issues += _listed(item.get("header_clarity_issues")) + issues += _issue_list(item, "header_clarity_issues") # Placeholder-only fields are the problem; that check lists no issues. if not issues and prompt_name != "placeholder_as_label": continue problem = "; ".join(str(issue) for issue in issues) - findings.append(( + findings.append(_Source( index, item, problem or str(item.get("reason") or "") or prompt_name, From eabee40934aea48d54a113b499baab948debafac Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 1 Oct 2026 18:46:56 -0400 Subject: [PATCH 066/126] docs(readme): add review prompt and move evaluator section to end --- README.md | 321 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 168 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index 6358712..80e03e8 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,159 @@ GEMINI_API_KEY=AIza... > checks only, no LLM findings, no CSV, and still a `200 OK` from the web app. > The only signal is `summary.dry_run` in the response. +## Running the Web App + +```bash +uv run python entry_points/api_server.py # http://localhost:8000 +``` + +Serves the UI and the audit API from one process. Users can paste their own API +key into the form instead of configuring one server-side; per-request keys take +priority over the environment. + +The audit endpoints stream **NDJSON** — progress events, one JSON object per +line, then a final `{"type":"result"}` object. Parsing that body with a single +`res.json()` fails; the front end branches on content type. + +To run it in a container: + +```bash +docker compose up --build # http://localhost:8000 +HOST_PORT=8789 docker compose up --build # if 8000 is taken +``` + +## Deployment + +Merges to `main` build the image, push it to `ghcr.io/c4g/va-dat`, and trigger a +Coolify deploy to . See [DEPLOY.md](DEPLOY.md) — in +particular the proxy settings, since response buffering breaks the progress +stream and short read timeouts cut off long audits. + +Note that Coolify runs the **image**; the hardening in `docker-compose.yml` +(`read_only`, `tmpfs`) applies to local runs only. + +## Continuous Integration + +`.github/workflows/ci.yml` runs on every PR to `main` and needs no API key — +everything it does is free: + +- `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock +- entry points import +- the tracked synthetic pytest suite (`uv run pytest -q`), with no private data or provider calls +- a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed +- `index.html`'s inline JavaScript parses +- the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit + +Default pytest discovery is limited to `tests/`, where `tests/test_evaluation.py` +covers the evaluator end to end through its CLI with a fake provider. Optional +detailed evaluator cases live in the ignored `.local-tests/evaluation/` folder; +run them with `uv run pytest -q .local-tests/evaluation`. CI never needs them. + +## Running the Pipeline + +### Dry run (no API calls, no cost) + +Generates all prompts and saves them as JSON files so you can inspect them before spending money: + +```bash +uv run python entry_points/run_pipeline.py --html test_files/dat_visionaid_home.html --dry-run +``` + +### Live run + +Sends prompts to the LLM and saves responses: + +```bash +uv run python entry_points/run_pipeline.py --html test_files/home.html +``` + +### Generate report + +After a live run, combine all findings into a single CSV: + +```bash +uv run python entry_points/generate_report.py +uv run python entry_points/generate_report.py --output-dir ./output --report-dir ./test_results/claude/ +``` + +### Pipeline options + +| Flag | Default | Description | +|------|---------|-------------| +| `--html` | (required) | Path to the HTML file to analyze | +| `--output-dir` | `./output` | Directory for results | +| `--model` | `claude-sonnet-5` | Anthropic model to use | +| `--dry-run` | off | Generate prompts without calling the API | +| `--include-summaries` | off | Include the 3 cross-cutting summary prompts | +| `--show-cost` | off | Print estimated dollar cost of the run based on model pricing | +| `--env-file` | `.env` | Path to environment file | + +### Report generator options + +| Flag | Default | Description | +|------|---------|-------------| +| `--output-dir` | `./output` | Directory containing pipeline output | +| `--report-dir` | `./test_results/claude/` | Directory to write the CSV report | + +## Output Structure + +### Pipeline output (`output/`) + +``` +output/ +├── manifest.json # Run metadata, token counts, prompt status +├── programmatic_findings.json # Rule-based checker results (free) +├── payloads/ # Raw extractor output (for inspection) +│ ├── cl01_payload.json +│ ├── cl02_payload.json +│ └── cl03_payload.json +└── prompts/ # One file per prompt + ├── page_title.json # Contains prompt text, payload slice, and API response + ├── heading_structure.json + ├── link_clarity.json + └── ... +``` + +### Report output (`test_results/claude/`) + +The report CSV has 13 columns matching the Vision Aid team's standard format: + +| Column | Description | +|--------|-------------| +| `ID` | Sequential row number | +| `element_name` | HTML element (e.g. ``, ` "link text"`) | +| `browser_combination` | Always `N/A` (static HTML analysis) | +| `page_title` | Page title from the analyzed HTML | +| `issue_title` | Short issue description | +| `steps_to_reproduce` | Element snippet or inspection steps | +| `actual_result` | What was found | +| `expected_result` | What WCAG requires | +| `recommendation` | Suggested fix | +| `wcag_sc` | WCAG success criterion (e.g. `1.1.1`) | +| `category` | Issue category (e.g. `Programmatic / Non-text Content`) | +| `log_date` | Date of the pipeline run | +| `reported_by` | `Programmatic` or the LLM model string | + +## Cost Estimate + +For visionaid.org homepage (using Claude Sonnet): + +| Approach | Input tokens | Cost | +|----------|-------------|------| +| Monolithic (entire HTML) | ~487,000 | ~$1.52 | +| Element-specific pipeline | ~18,000 | ~$0.32 | + +The pipeline skips prompts with empty payloads (e.g., no forms on the page = no form prompts), so actual cost varies by page content. + +## Attribution + +| Contributor | What they own | Key files | +|---|---|---| +| ahildebrandt3 | Extractors, programmatic checkers (CL01–CL03), CL01 prompts | `processing_scripts/llm_preprocessing/`, `processing_scripts/programmatic/` | +| Andrew Yin | CL02 + CL03 extractors, CL02 + CL03 prompts, LLM client, pipeline docs | `processing_scripts/llm_preprocessing/`, `processing_scripts/llm_client/`, `processing_scripts/llm/*.txt` | +| nfulton99 | HTML ingestion, packaging | `vision_aid/ingestion/pull_html.py`, `pyproject.toml` | +| ColeANiblett | Pipeline orchestration, prompt system, report generator | `processing_scripts/llm/{registry,slicers,templates}.py`, `entry_points/`, `docs/` | + ## Private Model Evaluation ### Choose an evaluation model @@ -477,6 +630,21 @@ you describe partial evidence in `review_notes`. For `unavailable_evidence` and `ambiguous` rows, leave both ID cells blank. The report command checks these rules and requires a `match_reason` whenever you enter an ID. +#### Using Codex or Claude Code for the review + +If you use Codex or Claude Code to fill out `review.csv` and generate the +report, give it the following prompt to guide semantic matching and avoid +incorrectly rejecting valid matches: + +```text +Review the latest run, fill out `review.csv`, and generate the report. Use practical semantic matching. +Count a match when a finding identifies the same underlying accessibility problem in the same element or relevant section, even if its wording, certainty, examples, or recommended fix differ from the Human audit. +Treat examples in a Human finding as illustrations of the problem unless they clearly describe separate defects. A different remediation recommendation does not invalidate detection of the problem. +Conditional or tentative findings can count when they identify the reported defect and the Human audit supplies the context needed to support that interpretation. Use the saved HTML to confirm the location. +Allow multiple findings together to cover a Human finding. Reject matches that merely share a WCAG criterion, concern a different element, or leave a distinct part of the reported problem undetected. +Record qualifications in `review_notes` rather than automatically withholding credit. Apply this interpretation of full coverage consistently. +``` + The run directory and Human audit are excluded from version control. This one-homepage result does not establish broad model equivalence. @@ -495,156 +663,3 @@ LLM prompts, and `review.csv`. Use the normalized file's `finding_id` values for programmatic matches. It makes no model requests and cannot produce a final report. `--max-cost-usd` must be positive but is not spent without `--live`. Start a new live run at step 3 to finish the evaluation. - -## Running the Web App - -```bash -uv run python entry_points/api_server.py # http://localhost:8000 -``` - -Serves the UI and the audit API from one process. Users can paste their own API -key into the form instead of configuring one server-side; per-request keys take -priority over the environment. - -The audit endpoints stream **NDJSON** — progress events, one JSON object per -line, then a final `{"type":"result"}` object. Parsing that body with a single -`res.json()` fails; the front end branches on content type. - -To run it in a container: - -```bash -docker compose up --build # http://localhost:8000 -HOST_PORT=8789 docker compose up --build # if 8000 is taken -``` - -## Deployment - -Merges to `main` build the image, push it to `ghcr.io/c4g/va-dat`, and trigger a -Coolify deploy to . See [DEPLOY.md](DEPLOY.md) — in -particular the proxy settings, since response buffering breaks the progress -stream and short read timeouts cut off long audits. - -Note that Coolify runs the **image**; the hardening in `docker-compose.yml` -(`read_only`, `tmpfs`) applies to local runs only. - -## Continuous Integration - -`.github/workflows/ci.yml` runs on every PR to `main` and needs no API key — -everything it does is free: - -- `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock -- entry points import -- the tracked synthetic pytest suite (`uv run pytest -q`), with no private data or provider calls -- a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed -- `index.html`'s inline JavaScript parses -- the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit - -Default pytest discovery is limited to `tests/`, where `tests/test_evaluation.py` -covers the evaluator end to end through its CLI with a fake provider. Optional -detailed evaluator cases live in the ignored `.local-tests/evaluation/` folder; -run them with `uv run pytest -q .local-tests/evaluation`. CI never needs them. - -## Running the Pipeline - -### Dry run (no API calls, no cost) - -Generates all prompts and saves them as JSON files so you can inspect them before spending money: - -```bash -uv run python entry_points/run_pipeline.py --html test_files/dat_visionaid_home.html --dry-run -``` - -### Live run - -Sends prompts to the LLM and saves responses: - -```bash -uv run python entry_points/run_pipeline.py --html test_files/home.html -``` - -### Generate report - -After a live run, combine all findings into a single CSV: - -```bash -uv run python entry_points/generate_report.py -uv run python entry_points/generate_report.py --output-dir ./output --report-dir ./test_results/claude/ -``` - -### Pipeline options - -| Flag | Default | Description | -|------|---------|-------------| -| `--html` | (required) | Path to the HTML file to analyze | -| `--output-dir` | `./output` | Directory for results | -| `--model` | `claude-sonnet-5` | Anthropic model to use | -| `--dry-run` | off | Generate prompts without calling the API | -| `--include-summaries` | off | Include the 3 cross-cutting summary prompts | -| `--show-cost` | off | Print estimated dollar cost of the run based on model pricing | -| `--env-file` | `.env` | Path to environment file | - -### Report generator options - -| Flag | Default | Description | -|------|---------|-------------| -| `--output-dir` | `./output` | Directory containing pipeline output | -| `--report-dir` | `./test_results/claude/` | Directory to write the CSV report | - -## Output Structure - -### Pipeline output (`output/`) - -``` -output/ -├── manifest.json # Run metadata, token counts, prompt status -├── programmatic_findings.json # Rule-based checker results (free) -├── payloads/ # Raw extractor output (for inspection) -│ ├── cl01_payload.json -│ ├── cl02_payload.json -│ └── cl03_payload.json -└── prompts/ # One file per prompt - ├── page_title.json # Contains prompt text, payload slice, and API response - ├── heading_structure.json - ├── link_clarity.json - └── ... -``` - -### Report output (`test_results/claude/`) - -The report CSV has 13 columns matching the Vision Aid team's standard format: - -| Column | Description | -|--------|-------------| -| `ID` | Sequential row number | -| `element_name` | HTML element (e.g. ``, ` "link text"`) | -| `browser_combination` | Always `N/A` (static HTML analysis) | -| `page_title` | Page title from the analyzed HTML | -| `issue_title` | Short issue description | -| `steps_to_reproduce` | Element snippet or inspection steps | -| `actual_result` | What was found | -| `expected_result` | What WCAG requires | -| `recommendation` | Suggested fix | -| `wcag_sc` | WCAG success criterion (e.g. `1.1.1`) | -| `category` | Issue category (e.g. `Programmatic / Non-text Content`) | -| `log_date` | Date of the pipeline run | -| `reported_by` | `Programmatic` or the LLM model string | - -## Cost Estimate - -For visionaid.org homepage (using Claude Sonnet): - -| Approach | Input tokens | Cost | -|----------|-------------|------| -| Monolithic (entire HTML) | ~487,000 | ~$1.52 | -| Element-specific pipeline | ~18,000 | ~$0.32 | - -The pipeline skips prompts with empty payloads (e.g., no forms on the page = no form prompts), so actual cost varies by page content. - -## Attribution - -| Contributor | What they own | Key files | -|---|---|---| -| ahildebrandt3 | Extractors, programmatic checkers (CL01–CL03), CL01 prompts | `processing_scripts/llm_preprocessing/`, `processing_scripts/programmatic/` | -| Andrew Yin | CL02 + CL03 extractors, CL02 + CL03 prompts, LLM client, pipeline docs | `processing_scripts/llm_preprocessing/`, `processing_scripts/llm_client/`, `processing_scripts/llm/*.txt` | -| nfulton99 | HTML ingestion, packaging | `vision_aid/ingestion/pull_html.py`, `pyproject.toml` | -| ColeANiblett | Pipeline orchestration, prompt system, report generator | `processing_scripts/llm/{registry,slicers,templates}.py`, `entry_points/`, `docs/` | From b2193457c3d6911131d2e36f7b0bd316a08e85ca Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 3 Oct 2026 19:09:02 -0400 Subject: [PATCH 067/126] docs(readme): clarify evaluator response handling --- README.md | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 80e03e8..5c14ec0 100644 --- a/README.md +++ b/README.md @@ -491,19 +491,21 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report ### How the evaluator reads LLM responses -The evaluator measures the tool as it is, so it reads each LLM response the -way the tool's CSV report does, using the tool's own code rather than a copy. - -**Parsing.** Each response is parsed with the tool's JSON parser, including its -code-fence and repair steps. A response the tool cannot parse, or whose shape -the check's rule cannot read (for example one object where a list is expected, -or text where `issues`, `vague_headings`, or `header_clarity_issues` should be -a list), is an Unreadable response: it gives no LLM findings, `run.json` lists the check -under `format_failures`, and the run continues with the next check. - -**What counts as a finding.** For the 12 checks the tool's CSV report covers, -the tool's own per-check rule decides, and each row it would write becomes one -LLM finding: +The evaluator reads each LLM response using va-dat's own parsing and reporting +code. This keeps the evaluation consistent with how va-dat produces its CSV +report. + +**Parsing.** The evaluator parses each response with va-dat's JSON parser, +including its code-fence and repair steps. If va-dat cannot parse a response, +or the check's rule cannot read its structure, it counts as an Unreadable +response. Examples include one object where a list is expected, or text where +`issues`, `vague_headings`, or `header_clarity_issues` should be a list. +An Unreadable response gives no LLM findings. The evaluator lists the check +under `format_failures` in `run.json` and continues with the next check. + +**What counts as a finding.** For the 12 checks covered by va-dat's CSV report, +the evaluator applies va-dat's per-check rules. Each row those rules would +produce becomes one LLM finding: - Page title and landmarks: one finding per entry in `issues`. Yes/no answers such as `is_descriptive` are ignored. @@ -517,18 +519,18 @@ LLM finding: with its issues joined. An item the model judged acceptable gives no finding, even if a yes/no answer such as `has_accessible_name` is `false`. -Each finding's problem, element, and location are the text the tool would -write in its row. `raw_source` keeps the full item, or for page title, +Each finding's problem, element, and location use the text va-dat would +write in its CSV row. `raw_source` keeps the full item, or for page title, headings, and landmarks the whole response plus the specific issue. -**What is skipped.** The tool's false-positive filter and LLM deduplication are -not applied. The filter guesses from the page HTML and can remove a finding -other than the one it matched; deduplication costs extra LLM calls and is not +**What is skipped.** The evaluator skips va-dat's false-positive filter and LLM +deduplication. The filter guesses from the page HTML and can remove a finding +other than the one it matched. Deduplication costs extra LLM calls and is not repeatable. Evaluation counts therefore measure what the model reported and -will not match the rows in the tool's downloadable CSV report. +will not match the rows in va-dat's downloadable CSV report. -**Checks shown on screen only.** Six checks have no rule in the tool's CSV -report; their results appear only in the web app's on-screen panel: +**Checks shown on screen only.** Six checks have no rule in va-dat's CSV +report. Their results appear only in the web app's on-screen panel: `table_semantics`, `placeholder_as_label`, `group_labels`, `form_instructions`, `complex_descriptions`, and `media_captions`. The evaluator still reads them, giving at most one finding per item, and only when the item's `issues` list is @@ -536,8 +538,8 @@ non-empty (for tables, `issues` or `header_clarity_issues`). Every `placeholder_as_label` item is a finding, because that check only receives placeholder-only fields. Yes/no answers never create a finding on their own. A table response may be one object or a list. These findings have -`"screen_only": true`, and `report.md` labels them "shown on screen only, not -in the tool's CSV report". +`"screen_only": true`. The evaluation report, `report.md`, labels them as shown +on screen only and absent from va-dat's CSV report. ### Evaluation run files From b83a09b63822840f9d972c9dafb3a02f13cfe209 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 3 Oct 2026 19:13:05 -0400 Subject: [PATCH 068/126] chore(readme): wording and formatting --- README.md | 148 +++++++++++++++++++++++++++--------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 5c14ec0..5719b47 100644 --- a/README.md +++ b/README.md @@ -119,11 +119,11 @@ HTML file (e.g. 1.9 MB) Each extractor has an `extract(file_path)` function that parses HTML with BeautifulSoup and returns a structured dict: -| Extractor | Focus | Output tokens (visionaid.org) | -|-----------|-------|-------------------------------| -| `semantic_checklist_01.py` | Page title, headings, links, landmarks, tables, iframes | ~17,600 | -| `forms_checklist_02.py` | Form fields with label source, instructions, required flags | ~2,500 | -| `nontext_checklist_03.py` | Images (4 categories), SVGs, icon fonts, video/audio | ~19,200 | +| Extractor | Focus | Output tokens (visionaid.org) | +| -------------------------- | ----------------------------------------------------------- | ----------------------------- | +| `semantic_checklist_01.py` | Page title, headings, links, landmarks, tables, iframes | ~17,600 | +| `forms_checklist_02.py` | Form fields with label source, instructions, required flags | ~2,500 | +| `nontext_checklist_03.py` | Images (4 categories), SVGs, icon fonts, video/audio | ~19,200 | These files live in `processing_scripts/llm_preprocessing/` and were authored by ahildebrandt3 and Andrew Yin. They should not need modification unless a new checklist (CL04+) is added. @@ -314,22 +314,22 @@ uv run python entry_points/generate_report.py --output-dir ./output --report-dir ### Pipeline options -| Flag | Default | Description | -|------|---------|-------------| -| `--html` | (required) | Path to the HTML file to analyze | -| `--output-dir` | `./output` | Directory for results | -| `--model` | `claude-sonnet-5` | Anthropic model to use | -| `--dry-run` | off | Generate prompts without calling the API | -| `--include-summaries` | off | Include the 3 cross-cutting summary prompts | -| `--show-cost` | off | Print estimated dollar cost of the run based on model pricing | -| `--env-file` | `.env` | Path to environment file | +| Flag | Default | Description | +| --------------------- | ----------------- | ------------------------------------------------------------- | +| `--html` | (required) | Path to the HTML file to analyze | +| `--output-dir` | `./output` | Directory for results | +| `--model` | `claude-sonnet-5` | Anthropic model to use | +| `--dry-run` | off | Generate prompts without calling the API | +| `--include-summaries` | off | Include the 3 cross-cutting summary prompts | +| `--show-cost` | off | Print estimated dollar cost of the run based on model pricing | +| `--env-file` | `.env` | Path to environment file | ### Report generator options -| Flag | Default | Description | -|------|---------|-------------| -| `--output-dir` | `./output` | Directory containing pipeline output | -| `--report-dir` | `./test_results/claude/` | Directory to write the CSV report | +| Flag | Default | Description | +| -------------- | ------------------------ | ------------------------------------ | +| `--output-dir` | `./output` | Directory containing pipeline output | +| `--report-dir` | `./test_results/claude/` | Directory to write the CSV report | ## Output Structure @@ -354,43 +354,43 @@ output/ The report CSV has 13 columns matching the Vision Aid team's standard format: -| Column | Description | -|--------|-------------| -| `ID` | Sequential row number | -| `element_name` | HTML element (e.g. ``, ` "link text"`) | -| `browser_combination` | Always `N/A` (static HTML analysis) | -| `page_title` | Page title from the analyzed HTML | -| `issue_title` | Short issue description | -| `steps_to_reproduce` | Element snippet or inspection steps | -| `actual_result` | What was found | -| `expected_result` | What WCAG requires | -| `recommendation` | Suggested fix | -| `wcag_sc` | WCAG success criterion (e.g. `1.1.1`) | -| `category` | Issue category (e.g. `Programmatic / Non-text Content`) | -| `log_date` | Date of the pipeline run | -| `reported_by` | `Programmatic` or the LLM model string | +| Column | Description | +| --------------------- | ---------------------------------------------------------- | +| `ID` | Sequential row number | +| `element_name` | HTML element (e.g. ``, ` "link text"`) | +| `browser_combination` | Always `N/A` (static HTML analysis) | +| `page_title` | Page title from the analyzed HTML | +| `issue_title` | Short issue description | +| `steps_to_reproduce` | Element snippet or inspection steps | +| `actual_result` | What was found | +| `expected_result` | What WCAG requires | +| `recommendation` | Suggested fix | +| `wcag_sc` | WCAG success criterion (e.g. `1.1.1`) | +| `category` | Issue category (e.g. `Programmatic / Non-text Content`) | +| `log_date` | Date of the pipeline run | +| `reported_by` | `Programmatic` or the LLM model string | ## Cost Estimate For visionaid.org homepage (using Claude Sonnet): -| Approach | Input tokens | Cost | -|----------|-------------|------| -| Monolithic (entire HTML) | ~487,000 | ~$1.52 | -| Element-specific pipeline | ~18,000 | ~$0.32 | +| Approach | Input tokens | Cost | +| ------------------------- | ------------ | ------ | +| Monolithic (entire HTML) | ~487,000 | ~$1.52 | +| Element-specific pipeline | ~18,000 | ~$0.32 | The pipeline skips prompts with empty payloads (e.g., no forms on the page = no form prompts), so actual cost varies by page content. ## Attribution -| Contributor | What they own | Key files | -|---|---|---| -| ahildebrandt3 | Extractors, programmatic checkers (CL01–CL03), CL01 prompts | `processing_scripts/llm_preprocessing/`, `processing_scripts/programmatic/` | -| Andrew Yin | CL02 + CL03 extractors, CL02 + CL03 prompts, LLM client, pipeline docs | `processing_scripts/llm_preprocessing/`, `processing_scripts/llm_client/`, `processing_scripts/llm/*.txt` | -| nfulton99 | HTML ingestion, packaging | `vision_aid/ingestion/pull_html.py`, `pyproject.toml` | -| ColeANiblett | Pipeline orchestration, prompt system, report generator | `processing_scripts/llm/{registry,slicers,templates}.py`, `entry_points/`, `docs/` | +| Contributor | What they own | Key files | +| ------------- | ---------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| ahildebrandt3 | Extractors, programmatic checkers (CL01–CL03), CL01 prompts | `processing_scripts/llm_preprocessing/`, `processing_scripts/programmatic/` | +| Andrew Yin | CL02 + CL03 extractors, CL02 + CL03 prompts, LLM client, pipeline docs | `processing_scripts/llm_preprocessing/`, `processing_scripts/llm_client/`, `processing_scripts/llm/*.txt` | +| nfulton99 | HTML ingestion, packaging | `vision_aid/ingestion/pull_html.py`, `pyproject.toml` | +| ColeANiblett | Pipeline orchestration, prompt system, report generator | `processing_scripts/llm/{registry,slicers,templates}.py`, `entry_points/`, `docs/` | -## Private Model Evaluation +## Model Evaluation ### Choose an evaluation model @@ -398,11 +398,11 @@ Each Evaluation run uses one fixed model through Anthropic's direct API. Choose it with `--model`; omitting the flag selects Haiku 4.5. The evaluator accepts only these exact IDs: -| Model | `--model` value | Thinking | Effort | -| --- | --- | --- | --- | +| Model | `--model` value | Thinking | Effort | +| ------------------ | --------------------------- | ------------------- | ------- | | Haiku 4.5, default | `claude-haiku-4-5-20251001` | 16,000-token budget | Omitted | -| Opus 5.5 | `claude-opus-5-5` | Adaptive | Medium | -| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | Medium | +| Opus 5.5 | `claude-opus-5-5` | Adaptive | Medium | +| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | Medium | For example, add `--model claude-opus-5-5` to the run command below to evaluate Opus. Friendly names and aliases are rejected. Thinking settings are fixed @@ -554,22 +554,22 @@ can read them. Move their supporting files into `artifacts/`, including the `prompts/` and `payloads/` directories, preserving filenames. There is no compatibility fallback or automatic migration. -| File | Role | -| --- | --- | -| `review.csv` | Original Human audit evidence, source sheet and row, stable IDs, and the decision columns the reviewer edits. | -| `snapshot.html` | The full Benchmark snapshot used to judge testability and coverage. | -| `normalized-llm-findings.json` | LLM findings with `finding_id` values for `review.csv`, read as described above. An Unreadable response can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | -| `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | -| `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | -| `report.md` | Scored results generated after successful reporting. | -| `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | -| `artifacts/pricing.json` | The versioned public model rates and capture date used to estimate this run's cost from reported usage. Older runs retain their original price schedule. | -| `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | -| `artifacts/raw-programmatic-findings.json` | Original output from the Programmatic checks. | -| `artifacts/prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | -| `artifacts/payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | -| `artifacts/payloads/cl02_payload.json` | Extracted forms, fields, and labels. | -| `artifacts/payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | +| File | Role | +| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `review.csv` | Original Human audit evidence, source sheet and row, stable IDs, and the decision columns the reviewer edits. | +| `snapshot.html` | The full Benchmark snapshot used to judge testability and coverage. | +| `normalized-llm-findings.json` | LLM findings with `finding_id` values for `review.csv`, read as described above. An Unreadable response can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | +| `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | +| `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | +| `report.md` | Scored results generated after successful reporting. | +| `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | +| `artifacts/pricing.json` | The versioned public model rates and capture date used to estimate this run's cost from reported usage. Older runs retain their original price schedule. | +| `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | +| `artifacts/raw-programmatic-findings.json` | Original output from the Programmatic checks. | +| `artifacts/prompts/*.json` | The exact `prompt_text` and filtered `payload_slice` prepared for each model request. These files do not hold model replies in an Evaluation run. | +| `artifacts/payloads/cl01_payload.json` | Extracted headings, links, landmarks, tables, and other semantic page structure. | +| `artifacts/payloads/cl02_payload.json` | Extracted forms, fields, and labels. | +| `artifacts/payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media. | The payload files show extracted page data before filtering. The prompt files show what the model actually received after filtering and slicing. To review a @@ -603,12 +603,12 @@ In particular, `source_sheet` and `source_row` point back to the Human audit, For **every row**, fill in `classification`: -| `classification` value | Use it when | -| --- | --- | -| `llm_eligible` | The Benchmark snapshot contains enough evidence to judge the defect, and the issue calls for the LLM's judgment. | -| `programmatic` | A Programmatic check can establish the defect from the available evidence. Classification alone does not mean the checker found it. | +| `classification` value | Use it when | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `llm_eligible` | The Benchmark snapshot contains enough evidence to judge the defect, and the issue calls for the LLM's judgment. | +| `programmatic` | A Programmatic check can establish the defect from the available evidence. Classification alone does not mean the checker found it. | | `unavailable_evidence` | The snapshot lacks evidence needed to verify the defect, such as behavior that requires interaction or a browser or assistive technology observation. | -| `ambiguous` | The Human finding is too unclear to decide fairly whether a finding covers it. | +| `ambiguous` | The Human finding is too unclear to decide fairly whether a finding covers it. | Base the classification on the Human finding and available evidence, not on whether this run happened to produce a matching finding. In @@ -618,12 +618,12 @@ optional, but `classification` cannot be blank when you generate the report. Then complete the remaining decision columns for that row: -| Column | What to enter | -| --- | --- | -| `llm_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-llm-findings.json` only when the selected LLM findings **jointly cover the whole Human finding**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | +| Column | What to enter | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `llm_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-llm-findings.json` only when the selected LLM findings **jointly cover the whole Human finding**. Separate multiple IDs with semicolons (`;`). Otherwise leave blank. | | `programmatic_finding_id` | Copy `finding_id` values from `$RUN_DIR/normalized-programmatic-findings.json` only when the selected Programmatic findings **jointly cover the whole Human finding**. Use semicolons for multiple IDs. Otherwise leave blank. | -| `match_reason` | If either finding-ID cell has an ID, briefly explain how those findings cover the full defect, including the relevant element or behavior. If both ID cells are blank, this can be blank. | -| `review_notes` | Optional context, such as a more precise location or a finding that covers only part of the defect. Notes do not award coverage. | +| `match_reason` | If either finding-ID cell has an ID, briefly explain how those findings cover the full defect, including the relevant element or behavior. If both ID cells are blank, this can be blank. | +| `review_notes` | Optional context, such as a more precise location or a finding that covers only part of the defect. Notes do not award coverage. | You may use findings from both files for one row. Each finding ID must exist in the named file and can be assigned to only one Human finding. Leave both ID From 9789af88cb33f56d525865de2379e6ffb4b68ef6 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 9 Oct 2026 12:25:28 -0400 Subject: [PATCH 069/126] fix(llm-client): honor explicit OpenAI retry policy --- processing_scripts/llm_client/audit.py | 5 ++++- tests/test_audit_request.py | 28 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py index 6869882..2266294 100644 --- a/processing_scripts/llm_client/audit.py +++ b/processing_scripts/llm_client/audit.py @@ -218,7 +218,10 @@ def __init__( elif self._is_openai: import openai - self._client = openai.OpenAI(api_key=api_key) + options: dict[str, Any] = {"api_key": api_key} + if max_retries is not None: + options["max_retries"] = max_retries + self._client = openai.OpenAI(**options) elif self._is_gemini: from google import genai diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py index 640dbe3..54a625e 100644 --- a/tests/test_audit_request.py +++ b/tests/test_audit_request.py @@ -405,3 +405,31 @@ def test_evaluation_can_disable_anthropic_sdk_retries(monkeypatch): max_retries=0, ) assert options == [{"api_key": "unused", "max_retries": 0}] + + +def test_evaluation_can_disable_openai_sdk_retries(monkeypatch): + """An explicit zero disables retries at the OpenAI SDK boundary.""" + from processing_scripts.llm_client.audit import AuditRequestClient + + options = [] + monkeypatch.setattr( + "openai.OpenAI", + lambda **kwargs: options.append(kwargs) or OpenAIFake(None), + ) + AuditRequestClient( + "unused", + request_config=AuditRequestConfig(model="gpt-6-luna"), + max_retries=0, + ) + assert options == [{"api_key": "unused", "max_retries": 0}] + + +def test_unspecified_openai_sdk_retries_preserve_sdk_default(monkeypatch): + """Production callers leave retry policy to the OpenAI SDK.""" + options = [] + monkeypatch.setattr( + "openai.OpenAI", + lambda **kwargs: options.append(kwargs) or OpenAIFake(None), + ) + PipelineClient("unused", "gpt-4o") + assert options == [{"api_key": "unused"}] From 4a9065d8cc65b36a38383aa19b4d12a2bd4799b9 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 9 Oct 2026 12:28:46 -0400 Subject: [PATCH 070/126] feat(evaluation): add GPT and Haiku model presets --- README.md | 47 +++++++----- tests/test_evaluation.py | 114 +++++++++++++++++++++++++++-- vision_aid/evaluation/cli.py | 21 ++++-- vision_aid/evaluation/llm_audit.py | 19 ++++- vision_aid/evaluation/pricing.json | 18 +++++ 5 files changed, 183 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 5719b47..2337375 100644 --- a/README.md +++ b/README.md @@ -394,22 +394,31 @@ The pipeline skips prompts with empty payloads (e.g., no forms on the page = no ### Choose an evaluation model -Each Evaluation run uses one fixed model through Anthropic's direct API. +Each Evaluation run uses one fixed model through Anthropic Messages or OpenAI +Chat Completions. Choose it with `--model`; omitting the flag selects Haiku 4.5. The evaluator accepts only these exact IDs: -| Model | `--model` value | Thinking | Effort | -| ------------------ | --------------------------- | ------------------- | ------- | -| Haiku 4.5, default | `claude-haiku-4-5-20251001` | 16,000-token budget | Omitted | -| Opus 5.5 | `claude-opus-5-5` | Adaptive | Medium | -| Sonnet 5.5 | `claude-sonnet-5-5` | Adaptive | Medium | - -For example, add `--model claude-opus-5-5` to the run command below to evaluate -Opus. Friendly names and aliases are rejected. Thinking settings are fixed -presets, and all three models use a 24,192-token output cap that includes -thinking and final text. Adaptive thinking does not reserve a fixed number of -tokens for final text. Temperature is omitted and summaries are disabled. -The selected model and settings are saved with each run. +| Model | `--model` value | Provider endpoint | Thinking / reasoning | Effort | +| --- | --- | --- | --- | --- | +| Haiku 4.5, default | `claude-haiku-4-5-20251001` | Anthropic Messages | 16,000-token budget | Omitted | +| Opus 5.5 | `claude-opus-5-5` | Anthropic Messages | Adaptive thinking | Medium | +| Sonnet 5.5 | `claude-sonnet-5-5` | Anthropic Messages | Adaptive thinking | Medium | +| Haiku 5.5 | `claude-haiku-5-5` | Anthropic Messages | Adaptive thinking | Medium | +| GPT-6 Luna | `gpt-6-luna` | OpenAI Chat Completions | Reasoning | Medium | +| GPT-6.1 Sol | `gpt-6.1-sol` | OpenAI Chat Completions | Reasoning | Medium | + +For example, add `--model gpt-6-luna` to the run command below to evaluate +GPT-6 Luna. Friendly names and aliases are rejected. Settings are fixed presets, +and all six models use a 24,192-token output cap that includes reasoning/thinking +and final text. Adaptive thinking does not reserve a fixed number of tokens for +final text. Matching effort labels and output caps do not imply equal computation +between providers. Temperature is omitted and summaries are disabled. The selected +model, provider, endpoint, and settings are saved with each run. + +Preview any preset without either API key by omitting `--live`. The preview shows +its provider, endpoint, settings, public token rates, and cost guardrail. Live +requests run sequentially without retries or fallback models. ### Get a final report with the supplied Human audit @@ -430,9 +439,10 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report if the browser builds the content, save rendered HTML there instead. Use `https://pristineai.com/` as its source URL. -2. **Set up live access and choose a limit.** Set `ANTHROPIC_API_KEY` in your - environment or the ignored repository-root `.env`. Set `MAX_COST_USD` to a - positive amount you approve. The limit is checked before each paid request; +2. **Set up live access and choose a limit.** Set `OPENAI_API_KEY` for GPT models + or `ANTHROPIC_API_KEY` for Claude models in your environment or the ignored + repository-root `.env`. Only the selected provider's key is required. + Set `MAX_COST_USD` to a positive amount you approve. The limit is checked before each paid request; a request in progress can put the final cost above it. 3. **Start a live Evaluation run.** Run this command in a terminal: @@ -447,8 +457,9 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report --live ``` - The CLI shows the selected model's thinking settings, effort, output token cap, - prompt count, token rates, pricing version, snapshot checksum, and limit. + The CLI shows the selected model's provider, endpoint, thinking/reasoning settings, + effort, output token cap, prompt count, token rates, pricing version, snapshot + checksum, and limit. The cost guardrail is checked between requests; a request underway can exceed it. Type `yes` to approve paid requests; `--approve-live` skips the prompt. It copies the snapshot, diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 877552e..670ff7b 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -14,7 +14,10 @@ from vision_aid.evaluation import llm_audit from vision_aid.evaluation.cli import main -MODELS = ("claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5") +MODELS = ( + "claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5", + "claude-haiku-5-5", "gpt-6-luna", "gpt-6.1-sol", +) CATEGORIES = [spec.name for spec in PROMPT_REGISTRY if not spec.is_summary] RUN_ROOT = {"review.csv", "snapshot.html", "normalized-programmatic-findings.json", "run.json", "artifacts"} @@ -127,10 +130,13 @@ def call(prompt: str) -> dict: return prompts -@pytest.mark.parametrize("model", [None, "claude-opus-5-5", "claude-sonnet-5-5"]) -def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, model): +@pytest.mark.parametrize("model", [None, *MODELS]) +def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, capsys, model): """Credentials alone never buy requests, and all homepage rows survive.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("provider access")) assert run_cli(workspace, *(["--model", model] if model else [])) == 0 run = only_run(workspace) assert {path.name for path in run.iterdir()} == RUN_ROOT @@ -139,7 +145,24 @@ def test_preview_preserves_evidence_without_provider_access(workspace, monkeypat "prompts", "payloads", } manifest = load(run, "run.json") - assert manifest["configuration"]["model"] == (model or MODELS[0]) + selected = model or MODELS[0] + config = manifest["configuration"] + assert config["model"] == selected + provider, endpoint = (("openai", "chat.completions") if selected.startswith("gpt-") + else ("anthropic", "messages")) + assert (config["provider"], config["endpoint"]) == (provider, endpoint) + assert config["temperature"] == "omitted" and config["summaries"] == "disabled" + assert config["max_output_tokens"] == 24192 + output = capsys.readouterr().out + assert f"provider: {provider}; endpoint: {endpoint}" in output + assert "Cost guardrail: $1" in output + if selected != MODELS[0]: + assert config["reasoning_effort"] == "medium" + if selected.startswith("gpt-"): + assert "thinking" not in config + assert "thinking: none" in output + else: + assert config["thinking"]["type"] == ("enabled" if selected == MODELS[0] else "adaptive") assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" assert len(CATEGORIES) == 18 assert [prompt["name"] for prompt in manifest["prompts"]] == CATEGORIES @@ -158,11 +181,14 @@ def test_preview_preserves_evidence_without_provider_access(workspace, monkeypat assert all(not row["classification"] for row in rows) +@pytest.mark.parametrize("model", MODELS) def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( - workspace, monkeypatch, capsys, + workspace, monkeypatch, capsys, model, ): """Declined approval prepares a preview; invalid models create no run.""" - monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: pytest.fail("paid call")) + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("paid call")) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("paid call")) with pytest.raises(SystemExit) as error: run_cli(workspace, "--model", "gpt-4o", "--live", "--approve-live") assert error.value.code == 2 and "invalid choice" in capsys.readouterr().err @@ -170,7 +196,7 @@ def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( monkeypatch.setattr("builtins.input", lambda _: "no") for interactive in (False, True): monkeypatch.setattr("sys.stdin", SimpleNamespace(isatty=lambda: interactive)) - assert run_cli(workspace, "--live") == 0 + assert run_cli(workspace, "--model", model, "--live") == 0 runs = list((workspace / ".model-evaluation" / "runs").iterdir()) assert len(runs) == 2 for run in runs: @@ -182,12 +208,15 @@ def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( (MODELS[0], {"type": "enabled", "budget_tokens": 16000}, None, "0.03915"), (MODELS[1], {"type": "adaptive"}, "medium", "0.1548"), (MODELS[2], {"type": "adaptive"}, "medium", "0.0783"), + ("claude-haiku-5-5", {"type": "adaptive"}, "medium", "0.003915"), ]) def test_presets_stream_every_category_and_report_saved_prices( workspace, monkeypatch, model, thinking, effort, cost, ): """Each preset's request settings, final text, model, and costs persist.""" requests, options = [], [] + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("wrong provider")) def create(**kwargs): """Stream thinking, then final text, with cache usage.""" @@ -248,6 +277,77 @@ def client(**kwargs): assert "9000 cached input tokens" in text and "1800 cache-creation input tokens" in text +@pytest.mark.parametrize("model,cost", [("gpt-6-luna", "0.0036"), ("gpt-6.1-sol", "0.072")]) +def test_gpt_presets_use_openai_key_and_save_response_identity( + workspace, monkeypatch, model, cost, +): + """GPT requests use the selected preset and preserve provider evidence for review.""" + requests, options = [], [] + monkeypatch.delenv("ANTHROPIC_API_KEY") + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("wrong provider")) + + def create(**kwargs): + """Return final text and reported usage through Chat Completions.""" + requests.append(kwargs) + return SimpleNamespace( + id="synthetic-gpt", model=f"{model}-provider", service_tier="default", + system_fingerprint="synthetic-fingerprint", + choices=[SimpleNamespace(message=SimpleNamespace( + content='[{"issues": ["Synthetic failure"], "location_hint": "body"}]'), + finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=1000, completion_tokens=200, + total_tokens=1200, prompt_tokens_details=None, + completion_tokens_details=SimpleNamespace(reasoning_tokens=150)), + ) + + def client(**kwargs): + """Record the SDK credentials and expose the provider request boundary.""" + options.append(kwargs) + return SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + + monkeypatch.setattr("openai.OpenAI", client) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert options == [{"api_key": "synthetic-openai", "max_retries": 0}] + assert len(requests) == len(CATEGORIES) + for request in requests: + assert request["model"] == model + assert request["reasoning_effort"] == "medium" + assert request["max_completion_tokens"] == 24192 + assert set(request) == {"model", "messages", "reasoning_effort", "max_completion_tokens"} + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["estimated_cost_usd"] == cost + raw = load(run, "artifacts/raw-llm-responses.json") + for item in raw: + result = item["result"] + assert result["model"] == f"{model}-provider" + assert result["provider"]["response_id"] == "synthetic-gpt" + assert result["provider"]["response_model"] == f"{model}-provider" + assert result["usage"]["provider_usage"]["completion_tokens_details"] == {"reasoning_tokens": 150} + assert result["usage"]["output_tokens"] == 200 + assert result["request"]["reasoning_effort"] == "medium" + assert "thinking" not in result["request"] + findings = load(run, "normalized-llm-findings.json") + assert findings and all(item["model"] == model for item in findings) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + text = report(run) + assert model in text and "18000 input tokens" in text and "3600 output tokens" in text + + +def test_run_help_explains_presets_and_provider_credentials(capsys): + """Operators can discover request settings and the relevant credentials in help.""" + with pytest.raises(SystemExit) as error: + main(["run", "--help"]) + assert error.value.code == 0 + text = capsys.readouterr().out + assert all(model in text for model in MODELS) + assert "Haiku 4.5" in text and "16,000" in text + assert "adaptive" in text and "medium" in text + assert "24,192" in text and "temperature" in text and "summaries" in text + assert "Chat Completions" in text + assert "OPENAI_API_KEY" in text and "ANTHROPIC_API_KEY" in text + def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): """Known decisions yield hand-calculated rates, cross-catches, and exclusions.""" fake_client( diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index fc3172c..2e63b49 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -41,9 +41,12 @@ def build_parser() -> argparse.ArgumentParser: run = commands.add_parser( "run", help="preview or execute an Evaluation run with one selected model", description=( - "Haiku uses a 16,000-token thinking budget; Opus and Sonnet use " - "adaptive thinking with medium effort. All presets omit temperature, " - "disable summaries, and cap output at 24,192 tokens including thinking." + "Haiku 4.5 uses a 16,000-token thinking budget; Haiku 5.5, Opus 5.5 " + "and Sonnet 5.5 use adaptive thinking with medium effort. GPT models " + "use Chat Completions with medium reasoning effort. All presets omit " + "temperature, disable summaries, and cap output at 24,192 tokens " + "including reasoning/thinking. Live GPT execution requires OPENAI_API_KEY; " + "live Claude execution requires ANTHROPIC_API_KEY. Previews need no key." ), ) run.add_argument( @@ -166,8 +169,9 @@ def _run(args: argparse.Namespace) -> int: write_json(run / "run.json", manifest) print(f"Evaluation run: {run}") print( - f"Model: {request_config.model}; endpoint: Messages; " - f"thinking: {configuration['thinking']['type']}; " + f"Model: {request_config.model}; provider: {configuration['provider']}; " + f"endpoint: {configuration['endpoint']}; " + f"thinking: {configuration.get('thinking', {}).get('type', 'none')}; " f"effort: {request_config.reasoning_effort or 'omitted'}; " f"thinking budget: {request_config.thinking_budget_tokens or 'none'}; " f"output cap: {request_config.max_output_tokens}" @@ -197,9 +201,12 @@ def _run(args: argparse.Namespace) -> int: print("Preview complete; no paid requests made.") return 0 load_dotenv(PROJECT_ROOT / ".env") - api_key = os.getenv("ANTHROPIC_API_KEY") + key_name = ( + "OPENAI_API_KEY" if configuration["provider"] == "openai" else "ANTHROPIC_API_KEY" + ) + api_key = os.getenv(key_name) if not api_key: - raise ValueError("Live execution requires ANTHROPIC_API_KEY") + raise ValueError(f"Live execution requires {key_name}") if not args.approve_live and ( not sys.stdin.isatty() or input("Type 'yes' to approve paid execution: ") != "yes" diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 5c12f08..6c4c78b 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -31,7 +31,16 @@ max_output_tokens=24192, temperature=None, ) - for model in ("claude-opus-5-5", "claude-sonnet-5-5") + for model in ("claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5") + }, + **{ + model: LLMRequestConfig( + model=model, + reasoning_effort="medium", + max_output_tokens=24192, + temperature=None, + ) + for model in ("gpt-6-luna", "gpt-6.1-sol") }, } USAGE_KEYS = ( @@ -44,11 +53,13 @@ def evaluation_configuration(config: LLMRequestConfig) -> dict[str, Any]: """Record the selected preset and evaluation-only request settings.""" + provider = "openai" if config.model in ("gpt-6-luna", "gpt-6.1-sol") else "anthropic" + metadata = config.as_metadata() return { "model": config.model, - "provider": "anthropic", - "endpoint": "messages", - "thinking": config.as_metadata()["thinking"], + "provider": provider, + "endpoint": "chat.completions" if provider == "openai" else "messages", + **({"thinking": metadata["thinking"]} if "thinking" in metadata else {}), "thinking_budget_tokens": config.thinking_budget_tokens, "reasoning_effort": config.reasoning_effort, "max_output_tokens": config.max_output_tokens, diff --git a/vision_aid/evaluation/pricing.json b/vision_aid/evaluation/pricing.json index c8fed30..2c6eac9 100644 --- a/vision_aid/evaluation/pricing.json +++ b/vision_aid/evaluation/pricing.json @@ -17,6 +17,24 @@ "output_per_million_usd": "10", "cached_input_per_million_usd": "0.20", "cache_creation_input_per_million_usd": "2.50" + }, + "gpt-6-luna": { + "input_per_million_usd": "0.10", + "output_per_million_usd": "0.50", + "cached_input_per_million_usd": "0.01", + "cache_creation_input_per_million_usd": "0.125" + }, + "gpt-6.1-sol": { + "input_per_million_usd": "2.00", + "output_per_million_usd": "10.00", + "cached_input_per_million_usd": "0.10", + "cache_creation_input_per_million_usd": "2.50" + }, + "claude-haiku-5-5": { + "input_per_million_usd": "0.10", + "output_per_million_usd": "0.50", + "cached_input_per_million_usd": "0.01", + "cache_creation_input_per_million_usd": "0.125" } }, "captured_at": "2026-10-01", From fa794087944c872961419552e22b9160fffeb4b2 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 9 Oct 2026 12:35:27 -0400 Subject: [PATCH 071/126] feat(evaluation): estimate costs per request and context tier --- README.md | 10 +- tests/test_evaluation.py | 250 ++++++++++++++++++++++++++++- vision_aid/evaluation/cli.py | 23 ++- vision_aid/evaluation/llm_audit.py | 31 +++- vision_aid/evaluation/pricing.json | 36 ++++- vision_aid/evaluation/pricing.py | 5 +- vision_aid/evaluation/review.py | 2 +- 7 files changed, 336 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2337375..633839b 100644 --- a/README.md +++ b/README.md @@ -461,6 +461,13 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report effort, output token cap, prompt count, token rates, pricing version, snapshot checksum, and limit. The cost guardrail is checked between requests; a request underway can exceed it. + Estimated run cost sums each request's reported token usage at its applicable + public price tier. GPT models use the higher tier above 272,000 total input + tokens; Haiku 5.5 uses it above 100,000. Equality uses the lower tier, and + input counts from separate requests are never combined to select a tier. + OpenAI cache reads and writes replace ordinary input charges, and completion + usage already includes reasoning. Anthropic's cache categories are separate + input counts; its write rate is for five-minute caching. Type `yes` to approve paid requests; `--approve-live` skips the prompt. It copies the snapshot, imports Human findings, runs checks, and saves responses and usage. @@ -475,7 +482,8 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report failure or cost limit and start a new live run. Failed requests are not retried. Use `$RUN_DIR/normalized-llm-findings.json` and `$RUN_DIR/normalized-programmatic-findings.json` for finding IDs and evidence. - `$RUN_DIR/artifacts/raw-llm-responses.json` has model responses; + `$RUN_DIR/artifacts/raw-llm-responses.json` has model responses, unchanged raw + provider usage, and each request's billable categories and estimated cost; `$RUN_DIR/artifacts/raw-programmatic-findings.json` has the original checker output. 5. **Make the human decisions in one CSV.** Open `$RUN_DIR/review.csv` in diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 670ff7b..a3d643d 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -130,6 +130,59 @@ def call(prompt: str) -> dict: return prompts +def recording_provider_usage(monkeypatch, model, *usages, stream_failure_at=None): + """Replay SDK usage payloads, then zero usage, through the public CLI.""" + requests = [] + text = '{"issues": ["Synthetic failure"]}' + + def create(**kwargs): + """Return a Chat Completion or streamed Message with supplied usage.""" + index = len(requests) + requests.append(kwargs) + usage = usages[index] if index < len(usages) else {} + if model.startswith("gpt-"): + return SimpleNamespace( + id="synthetic-price", model=model, + choices=[SimpleNamespace(message=SimpleNamespace(content=text), finish_reason="stop")], + usage=SimpleNamespace( + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("prompt_tokens", 0) + usage.get("completion_tokens", 0), + prompt_tokens_details=usage.get("prompt_tokens_details"), + completion_tokens_details=usage.get("completion_tokens_details"), + ), + ) + + def events(): + """Expose reported usage before optional transport failure.""" + yield SimpleNamespace(type="message_start", message=SimpleNamespace( + id="synthetic-price", model=model, type="message", usage=SimpleNamespace( + input_tokens=usage.get("input_tokens", 0), output_tokens=0, + cache_read_input_tokens=usage.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage.get("cache_creation_input_tokens", 0), + ))) + yield SimpleNamespace(type="content_block_start", content_block=SimpleNamespace( + type="text", text=text)) + yield SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), + usage=SimpleNamespace(output_tokens=usage.get("output_tokens", 0))) + if index == stream_failure_at: + raise TimeoutError("synthetic stream failure") + yield SimpleNamespace(type="message_stop") + + return nullcontext(events()) + + if model.startswith("gpt-"): + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("wrong provider")) + monkeypatch.setattr("openai.OpenAI", lambda **_: SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)))) + else: + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("wrong provider")) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: SimpleNamespace( + messages=SimpleNamespace(create=create))) + return requests + + @pytest.mark.parametrize("model", [None, *MODELS]) def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, capsys, model): """Credentials alone never buy requests, and all homepage rows survive.""" @@ -272,7 +325,7 @@ def client(**kwargs): edit_review(run, *[{"classification": "llm_eligible"}] * 6) text = report(run) assert model in text and run.name in text - assert "published prices captured on 2026-10-01" in text + assert "published prices captured on 2026-10-09" in text assert "18000 input tokens" in text and "3600 output tokens" in text assert "9000 cached input tokens" in text and "1800 cache-creation input tokens" in text @@ -348,6 +401,201 @@ def test_run_help_explains_presets_and_provider_credentials(capsys): assert "Chat Completions" in text assert "OPENAI_API_KEY" in text and "ANTHROPIC_API_KEY" in text + +@pytest.mark.parametrize("model,threshold,low,high,source", [ + ("gpt-6-luna", 272000, ("0.10", "0.50", "0.01", "0.125"), + ("0.20", "0.75", "0.02", "0.25"), "https://developers.openai.com/api/docs/pricing"), + ("gpt-6.1-sol", 272000, ("2.00", "10.00", "0.10", "2.50"), + ("4.00", "15.00", "0.20", "5.00"), "https://developers.openai.com/api/docs/pricing"), + ("claude-haiku-5-5", 100000, ("0.10", "0.50", "0.01", "0.125"), + ("0.50", "2.50", "0.05", "0.625"), + "https://platform.claude.com/docs/en/about-claude/pricing"), +]) +def test_preview_saves_and_displays_both_request_price_tiers( + workspace, monkeypatch, capsys, model, threshold, low, high, source, +): + """Key-free previews expose the recorded rates applicable to each request.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("provider access")) + assert run_cli(workspace, "--model", model) == 0 + run = only_run(workspace) + schedule = load(run, "artifacts/pricing.json") + assert schedule["captured_at"] == "2026-10-09" + assert schedule["version"] == "public-standard-2026-10-09" + assert source in schedule["sources"] + manifest = load(run, "run.json") + assert manifest["pricing_sha256"] == hashlib.sha256( + (run / "artifacts/pricing.json").read_bytes() + ).hexdigest() + output = capsys.readouterr().out + assert f"up to {threshold:,} total input tokens per request" in output + assert f"above {threshold:,} total input tokens per request" in output + cache_write = "5-minute cache write" if model.startswith("claude-") else "cache write" + for input_rate, output_rate, read_rate, write_rate in (low, high): + assert (f"input ${input_rate}; output ${output_rate}; " + f"cache read ${read_rate}; {cache_write} ${write_rate}") in output + if model.startswith("gpt-"): + assert "5-minute" not in output + + +@pytest.mark.parametrize("model,cost", [("gpt-6-luna", "0.0002345"), ("gpt-6.1-sol", "0.00467")]) +def test_openai_prices_cached_subsets_and_reasoning_once(workspace, monkeypatch, model, cost): + """Cache reads/writes replace ordinary input; output already includes reasoning.""" + details = SimpleNamespace(cached_tokens=200, cache_write_tokens=100) + requests = recording_provider_usage(monkeypatch, model, { + "prompt_tokens": 1000, "completion_tokens": 300, "prompt_tokens_details": details, + "completion_tokens_details": SimpleNamespace(reasoning_tokens=250), + }) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert len(requests) == len(CATEGORIES) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["estimated_cost_usd"] == cost + # Aggregate reported usage and raw provider evidence retain their existing meaning. + assert manifest["usage"] == {"input_tokens": 1000, "cached_input_tokens": 200, + "cache_creation_input_tokens": 0, "output_tokens": 300} + (first, *rest) = load(run, "artifacts/raw-llm-responses.json") + assert first["result"]["usage"]["provider_usage"]["prompt_tokens_details"] == { + "cached_tokens": 200, "cache_write_tokens": 100, + } + assert first["result"]["usage"]["provider_usage"]["completion_tokens_details"] == { + "reasoning_tokens": 250, + } + assert first["billable_usage"] == {"input_tokens": 700, "cached_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 300} + assert first["estimated_cost_usd"] == cost + assert all(item["estimated_cost_usd"] == "0" for item in rest) + + +@pytest.mark.parametrize("model,threshold,costs,total", [ + ("gpt-6-luna", 272000, ["0.0271895", "0.0543767"], "0.0815662"), + ("gpt-6.1-sol", 272000, ["0.54377", "1.087494"], "1.631264"), + ("claude-haiku-5-5", 100000, ["0.0099895", "0.049948"], "0.0599375"), +]) +def test_each_request_uses_total_input_and_inclusive_price_threshold( + workspace, monkeypatch, model, threshold, costs, total, +): + """Equality gets the lower tier; one extra token prices the whole request higher.""" + if model.startswith("gpt-"): + usages = [{"prompt_tokens": size, "completion_tokens": 10, + "prompt_tokens_details": SimpleNamespace(cached_tokens=200, cache_write_tokens=100)} + for size in (threshold, threshold + 1)] + else: + usages = [{"input_tokens": size - 300, "output_tokens": 10, + "cache_read_input_tokens": 200, "cache_creation_input_tokens": 100} + for size in (threshold, threshold + 1)] + requests = recording_provider_usage(monkeypatch, model, *usages) + assert run_cli(workspace, "--model", model, "--max-cost-usd", "10", + "--live", "--approve-live") == 0 + assert len(requests) == len(CATEGORIES) + run = only_run(workspace) + assert load(run, "run.json")["estimated_cost_usd"] == total + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw[:2]] == costs + assert [item["billable_usage"]["input_tokens"] for item in raw[:2]] == [threshold - 300, threshold - 299] + assert all(item["estimated_cost_usd"] == "0" for item in raw[2:]) + + +@pytest.mark.parametrize("limited", [False, True]) +@pytest.mark.parametrize("model,short_size,long_size,costs,total,partial,guardrail", [ + ("gpt-6-luna", 136001, 272001, + ["0.0136051", "0.0136051", "0.0544077", "0.000015"], "0.0816329", "0.0816179", "0.03"), + ("gpt-6.1-sol", 136001, 272001, + ["0.272102", "0.272102", "1.088154", "0.0003"], "1.632658", "1.632358", "0.6"), + ("claude-haiku-5-5", 50001, 100001, + ["0.0050051", "0.0050051", "0.0500255", "0.000015"], "0.0600507", "0.0600357", "0.012"), +]) +def test_mixed_request_sizes_accumulate_cost_and_drive_guardrail( + workspace, monkeypatch, model, short_size, long_size, costs, total, partial, guardrail, limited, +): + """Combined short input never triggers long rates or stops execution early.""" + input_key, output_key = (("prompt_tokens", "completion_tokens") if model.startswith("gpt-") + else ("input_tokens", "output_tokens")) + usages = [{input_key: size, output_key: 10} for size in (short_size, short_size, long_size, 100)] + requests = recording_provider_usage(monkeypatch, model, *usages) + assert run_cli(workspace, "--model", model, "--max-cost-usd", guardrail if limited else "10", + "--live", "--approve-live") == (1 if limited else 0) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["estimated_cost_usd"] == (partial if limited else total) + assert len(requests) == (3 if limited else len(CATEGORIES)) + assert manifest["incomplete_reason"] == ("cost guardrail exhausted" if limited else None) + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw[:4]] == (costs[:3] if limited else costs) + assert manifest["usage"]["input_tokens"] == short_size * 2 + long_size + (0 if limited else 100) + assert manifest["usage"]["output_tokens"] == (30 if limited else 40) + assert load(run, "normalized-llm-findings.json") + + +def test_stream_failure_retains_observed_tiered_cost_and_partial_usage(workspace, monkeypatch, capsys): + """A failed stream still contributes observed input, caches, and output to cost.""" + model = "claude-haiku-5-5" + requests = recording_provider_usage(monkeypatch, model, + {"input_tokens": 1000, "output_tokens": 10}, + {"input_tokens": 99701, "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 10}, + stream_failure_at=1, + ) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 1 + assert len(requests) == 2 + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["incomplete_reason"] == "synthetic stream failure" + assert manifest["estimated_cost_usd"] == "0.050053" + assert manifest["usage"] == {"input_tokens": 100701, "cached_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 20} + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw] == ["0.000105", "0.049948"] + assert raw[1]["result"]["response"] == '{"issues": ["Synthetic failure"]}' + assert raw[1]["result"]["failure"]["type"] == "TimeoutError" + assert raw[1]["result"]["usage"]["provider_usage"] == { + "input_tokens": 99701, "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 10, + } + assert len(load(run, "normalized-llm-findings.json")) == 1 + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + assert "Incomplete" in refused(run, capsys) + + +def test_historical_reports_keep_saved_prices_and_stored_estimate(workspace, monkeypatch): + """Legacy evidence needs no new per-request fields or current schedule rates.""" + response = {"success": True, "response": '{"issues": []}', + "usage": {"input_tokens": 10000, "output_tokens": 1000}} + fake_client(monkeypatch, *[response] * len(CATEGORIES)) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + # Turn the saved artifacts into a historical fixture with its original rates. + schedule = { + "version": "historical-public-2025-10-01", "published_at": "2025-10-01", + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "models": {MODELS[0]: { + "input_per_million_usd": "20", "output_per_million_usd": "100", + "cached_input_per_million_usd": "2", "cache_creation_input_per_million_usd": "25", + }}, + } + price_path = run / "artifacts/pricing.json" + price_path.write_text(json.dumps(schedule)) + manifest = load(run, "run.json") + manifest.update(pricing_version=schedule["version"], + pricing_sha256=hashlib.sha256(price_path.read_bytes()).hexdigest(), + estimated_cost_usd="5.4") + (run / "run.json").write_text(json.dumps(manifest)) + raw = load(run, "artifacts/raw-llm-responses.json") + for item in raw: + item.pop("estimated_cost_usd") + item.pop("billable_usage") + (run / "artifacts/raw-llm-responses.json").write_text(json.dumps(raw)) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + originals = {path: path.read_bytes() for path in (price_path, run / "run.json", + run / "artifacts/raw-llm-responses.json")} + text = report(run) + assert "Estimated run cost: $5.40" in text + assert "published prices dated 2025-10-01" in text + assert "180000 input tokens" in text and "18000 output tokens" in text + assert all(path.read_bytes() == saved for path, saved in originals.items()) + def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): """Known decisions yield hand-calculated rates, cross-catches, and exclusions.""" fake_client( diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 2e63b49..8eb65ae 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -186,12 +186,25 @@ def _run(args: argparse.Namespace) -> int: ) print(f"Pricing: {schedule.version} ({schedule.identity})") rates = schedule.data["models"][request_config.model] - print( - f"Rates per million tokens (USD): input ${rates['input_per_million_usd']}; " - f"output ${rates['output_per_million_usd']}; " - f"cache read ${rates['cached_input_per_million_usd']}; " - f"5-minute cache write ${rates['cache_creation_input_per_million_usd']}" + tiers = [("", rates)] + if "long_context" in rates: + upper = rates["long_context"] + threshold = upper["above_input_tokens"] + tiers = [ + (f"up to {threshold:,} total input tokens per request", rates), + (f"above {threshold:,} total input tokens per request", upper), + ] + cache_write = ( + "5-minute cache write" if configuration["provider"] == "anthropic" else "cache write" ) + for description, tier in tiers: + print( + f"Rates per million tokens (USD){f' ({description})' if description else ''}: " + f"input ${tier['input_per_million_usd']}; " + f"output ${tier['output_per_million_usd']}; " + f"cache read ${tier['cached_input_per_million_usd']}; " + f"{cache_write} ${tier['cache_creation_input_per_million_usd']}" + ) print( f"Cost guardrail: ${limit}; checked before each request; " "a request underway can exceed the limit" diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 6c4c78b..11fa02f 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -87,6 +87,20 @@ def positive_cost(value: str) -> Decimal: return amount +def billable_usage(model: str, reported: dict[str, Any]) -> dict[str, int]: + """Separate OpenAI cached subsets without changing reported evidence.""" + usage = {key: int(reported.get(key) or 0) for key in USAGE_KEYS} + if model in ("gpt-6-luna", "gpt-6.1-sol"): + raw = reported.get("provider_usage") or {} + details = raw.get("prompt_tokens_details") or {} + usage["cached_input_tokens"] = int(details.get("cached_tokens") or 0) + usage["cache_creation_input_tokens"] = int(details.get("cache_write_tokens") or 0) + usage["input_tokens"] -= ( + usage["cached_input_tokens"] + usage["cache_creation_input_tokens"] + ) + return usage + + def execute( run: Path, manifest: dict[str, Any], api_key: str, request_config: LLMRequestConfig, @@ -100,15 +114,14 @@ def execute( max_retries=0, ) usage: dict[str, int] = {key: 0 for key in USAGE_KEYS} + estimated_cost = Decimal(0) responses: list[dict[str, Any]] = [] findings: list[dict[str, Any]] = [] format_failures: list[str] = [] complete = True reason = None for prompt in manifest["prompts"]: - if Decimal(schedule.estimate(request_config.model, usage)) >= Decimal( - manifest["max_cost_usd"] - ): + if estimated_cost >= Decimal(manifest["max_cost_usd"]): complete, reason = False, "cost guardrail exhausted" break name = prompt["name"] @@ -129,7 +142,13 @@ def execute( reported: dict[str, Any] = result.get("usage") or {} for key in USAGE_KEYS: usage[key] += int(reported.get(key) or 0) - responses.append({"prompt": name, "result": result}) + billable = billable_usage(request_config.model, reported) + request_cost = schedule.estimate(request_config.model, billable) + estimated_cost += Decimal(request_cost) + responses.append({ + "prompt": name, "result": result, "billable_usage": billable, + "estimated_cost_usd": request_cost, + }) if result.get("success"): normalized = normalize_prompt_response( name, @@ -145,7 +164,7 @@ def execute( complete, reason = False, str(result.get("error") or "request failed") manifest.update( usage=usage, - estimated_cost_usd=schedule.estimate(request_config.model, usage), + estimated_cost_usd=format(estimated_cost.normalize(), "f"), complete=False, incomplete_reason=reason, format_failures=format_failures, @@ -158,7 +177,7 @@ def execute( manifest.update( complete=complete, incomplete_reason=reason, - estimated_cost_usd=schedule.estimate(request_config.model, usage), + estimated_cost_usd=format(estimated_cost.normalize(), "f"), ) write_json(run / "run.json", manifest) return complete diff --git a/vision_aid/evaluation/pricing.json b/vision_aid/evaluation/pricing.json index 2c6eac9..653c99c 100644 --- a/vision_aid/evaluation/pricing.json +++ b/vision_aid/evaluation/pricing.json @@ -22,22 +22,46 @@ "input_per_million_usd": "0.10", "output_per_million_usd": "0.50", "cached_input_per_million_usd": "0.01", - "cache_creation_input_per_million_usd": "0.125" + "cache_creation_input_per_million_usd": "0.125", + "long_context": { + "above_input_tokens": 272000, + "input_per_million_usd": "0.20", + "output_per_million_usd": "0.75", + "cached_input_per_million_usd": "0.02", + "cache_creation_input_per_million_usd": "0.25" + } }, "gpt-6.1-sol": { "input_per_million_usd": "2.00", "output_per_million_usd": "10.00", "cached_input_per_million_usd": "0.10", - "cache_creation_input_per_million_usd": "2.50" + "cache_creation_input_per_million_usd": "2.50", + "long_context": { + "above_input_tokens": 272000, + "input_per_million_usd": "4.00", + "output_per_million_usd": "15.00", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "5.00" + } }, "claude-haiku-5-5": { "input_per_million_usd": "0.10", "output_per_million_usd": "0.50", "cached_input_per_million_usd": "0.01", - "cache_creation_input_per_million_usd": "0.125" + "cache_creation_input_per_million_usd": "0.125", + "long_context": { + "above_input_tokens": 100000, + "input_per_million_usd": "0.50", + "output_per_million_usd": "2.50", + "cached_input_per_million_usd": "0.05", + "cache_creation_input_per_million_usd": "0.625" + } } }, - "captured_at": "2026-10-01", - "source": "https://platform.claude.com/docs/en/about-claude/pricing", - "version": "anthropic-public-2026-10-01" + "captured_at": "2026-10-09", + "sources": [ + "https://developers.openai.com/api/docs/pricing", + "https://platform.claude.com/docs/en/about-claude/pricing" + ], + "version": "public-standard-2026-10-09" } diff --git a/vision_aid/evaluation/pricing.py b/vision_aid/evaluation/pricing.py index 1985000..34d25a0 100644 --- a/vision_aid/evaluation/pricing.py +++ b/vision_aid/evaluation/pricing.py @@ -48,7 +48,7 @@ def price_description(self) -> str: return f"published prices dated {self.data['published_at']}" def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: - """Price reported categories without rounding away tie precision.""" + """Price one request's disjoint categories at its input-size tier.""" try: rates = self.data["models"][model] except KeyError as error: @@ -61,6 +61,9 @@ def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: usage.get("cache_creation_input_tokens", 0) or 0 ) output_tokens = Decimal(usage.get("output_tokens", 0) or 0) + upper = rates.get("long_context") + if upper and input_tokens + cached_tokens + cache_creation_tokens > upper["above_input_tokens"]: + rates = upper cost = ( input_tokens * Decimal(rates["input_per_million_usd"]) + cached_tokens * Decimal(rates["cached_input_per_million_usd"]) diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index c2b90bb..49cc805 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -189,7 +189,7 @@ def write_report( lines.extend([ _rate("Overall", len(covered), len(rows)), "", - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from {', '.join(tokens)} multiplied by the {_md(pricing_description)}. The actual bill may differ slightly.", + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated using provider-reported token usage and the {_md(pricing_description)}. Reported totals: {', '.join(tokens)}. The actual bill may differ slightly.", "", ]) if manifest["format_failures"]: From 8fd8e3f069c12926bca6d1075a0690c591823222 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 9 Oct 2026 12:37:48 -0400 Subject: [PATCH 072/126] refactor(evaluation): centralize OpenAI preset membership --- vision_aid/evaluation/llm_audit.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 11fa02f..6feb828 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -16,6 +16,7 @@ from vision_aid.evaluation.pricing import PriceSchedule DEFAULT_MODEL = "claude-haiku-4-5-20251001" +OPENAI_MODELS = ("gpt-6-luna", "gpt-6.1-sol") MODEL_PRESETS = { DEFAULT_MODEL: LLMRequestConfig( model=DEFAULT_MODEL, @@ -40,7 +41,7 @@ max_output_tokens=24192, temperature=None, ) - for model in ("gpt-6-luna", "gpt-6.1-sol") + for model in OPENAI_MODELS }, } USAGE_KEYS = ( @@ -53,7 +54,7 @@ def evaluation_configuration(config: LLMRequestConfig) -> dict[str, Any]: """Record the selected preset and evaluation-only request settings.""" - provider = "openai" if config.model in ("gpt-6-luna", "gpt-6.1-sol") else "anthropic" + provider = "openai" if config.model in OPENAI_MODELS else "anthropic" metadata = config.as_metadata() return { "model": config.model, @@ -90,7 +91,7 @@ def positive_cost(value: str) -> Decimal: def billable_usage(model: str, reported: dict[str, Any]) -> dict[str, int]: """Separate OpenAI cached subsets without changing reported evidence.""" usage = {key: int(reported.get(key) or 0) for key in USAGE_KEYS} - if model in ("gpt-6-luna", "gpt-6.1-sol"): + if model in OPENAI_MODELS: raw = reported.get("provider_usage") or {} details = raw.get("prompt_tokens_details") or {} usage["cached_input_tokens"] = int(details.get("cached_tokens") or 0) From a9c73b9b1c19f1cbac096449f4d3bcf1a1b98cef Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:23:04 -0400 Subject: [PATCH 073/126] feat(evaluation): default to Haiku 5.5 and remove Haiku 4.5 --- README.md | 7 +++---- tests/test_evaluation.py | 20 +++++++++----------- vision_aid/evaluation/cli.py | 2 +- vision_aid/evaluation/llm_audit.py | 10 ++-------- vision_aid/evaluation/pricing.json | 6 ------ 5 files changed, 15 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 633839b..4f33794 100644 --- a/README.md +++ b/README.md @@ -396,21 +396,20 @@ The pipeline skips prompts with empty payloads (e.g., no forms on the page = no Each Evaluation run uses one fixed model through Anthropic Messages or OpenAI Chat Completions. -Choose it with `--model`; omitting the flag selects Haiku 4.5. The evaluator +Choose it with `--model`; omitting the flag selects Haiku 5.5. The evaluator accepts only these exact IDs: | Model | `--model` value | Provider endpoint | Thinking / reasoning | Effort | | --- | --- | --- | --- | --- | -| Haiku 4.5, default | `claude-haiku-4-5-20251001` | Anthropic Messages | 16,000-token budget | Omitted | +| Haiku 5.5, default | `claude-haiku-5-5` | Anthropic Messages | Adaptive thinking | Medium | | Opus 5.5 | `claude-opus-5-5` | Anthropic Messages | Adaptive thinking | Medium | | Sonnet 5.5 | `claude-sonnet-5-5` | Anthropic Messages | Adaptive thinking | Medium | -| Haiku 5.5 | `claude-haiku-5-5` | Anthropic Messages | Adaptive thinking | Medium | | GPT-6 Luna | `gpt-6-luna` | OpenAI Chat Completions | Reasoning | Medium | | GPT-6.1 Sol | `gpt-6.1-sol` | OpenAI Chat Completions | Reasoning | Medium | For example, add `--model gpt-6-luna` to the run command below to evaluate GPT-6 Luna. Friendly names and aliases are rejected. Settings are fixed presets, -and all six models use a 24,192-token output cap that includes reasoning/thinking +and all five models use a 24,192-token output cap that includes reasoning/thinking and final text. Adaptive thinking does not reserve a fixed number of tokens for final text. Matching effort labels and output caps do not imply equal computation between providers. Temperature is omitted and summaries are disabled. The selected diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index a3d643d..37fe897 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -15,8 +15,8 @@ from vision_aid.evaluation.cli import main MODELS = ( - "claude-haiku-4-5-20251001", "claude-opus-5-5", "claude-sonnet-5-5", - "claude-haiku-5-5", "gpt-6-luna", "gpt-6.1-sol", + "claude-haiku-5-5", "claude-opus-5-5", "claude-sonnet-5-5", + "gpt-6-luna", "gpt-6.1-sol", ) CATEGORIES = [spec.name for spec in PROMPT_REGISTRY if not spec.is_summary] RUN_ROOT = {"review.csv", "snapshot.html", "normalized-programmatic-findings.json", @@ -209,13 +209,12 @@ def test_preview_preserves_evidence_without_provider_access(workspace, monkeypat output = capsys.readouterr().out assert f"provider: {provider}; endpoint: {endpoint}" in output assert "Cost guardrail: $1" in output - if selected != MODELS[0]: - assert config["reasoning_effort"] == "medium" + assert config["reasoning_effort"] == "medium" if selected.startswith("gpt-"): assert "thinking" not in config assert "thinking: none" in output else: - assert config["thinking"]["type"] == ("enabled" if selected == MODELS[0] else "adaptive") + assert config["thinking"]["type"] == "adaptive" assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" assert len(CATEGORIES) == 18 assert [prompt["name"] for prompt in manifest["prompts"]] == CATEGORIES @@ -258,10 +257,9 @@ def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( @pytest.mark.parametrize("model,thinking,effort,cost", [ - (MODELS[0], {"type": "enabled", "budget_tokens": 16000}, None, "0.03915"), + (MODELS[0], {"type": "adaptive"}, "medium", "0.003915"), (MODELS[1], {"type": "adaptive"}, "medium", "0.1548"), (MODELS[2], {"type": "adaptive"}, "medium", "0.0783"), - ("claude-haiku-5-5", {"type": "adaptive"}, "medium", "0.003915"), ]) def test_presets_stream_every_category_and_report_saved_prices( workspace, monkeypatch, model, thinking, effort, cost, @@ -395,7 +393,7 @@ def test_run_help_explains_presets_and_provider_credentials(capsys): assert error.value.code == 0 text = capsys.readouterr().out assert all(model in text for model in MODELS) - assert "Haiku 4.5" in text and "16,000" in text + assert "Haiku 5.5 (default)" in text assert "adaptive" in text and "medium" in text assert "24,192" in text and "temperature" in text and "summaries" in text assert "Chat Completions" in text @@ -560,7 +558,7 @@ def test_stream_failure_retains_observed_tiered_cost_and_partial_usage(workspace def test_historical_reports_keep_saved_prices_and_stored_estimate(workspace, monkeypatch): - """Legacy evidence needs no new per-request fields or current schedule rates.""" + """Saved evidence reports with its original prices and stored estimate.""" response = {"success": True, "response": '{"issues": []}', "usage": {"input_tokens": 10000, "output_tokens": 1000}} fake_client(monkeypatch, *[response] * len(CATEGORIES)) @@ -664,8 +662,8 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): @pytest.mark.parametrize("failure,calls,cost,reason", [ - (True, 2, "0.2002", "synthetic failure"), - (False, 1, "0.1001", "cost guardrail exhausted"), + (True, 2, "0.02002", "synthetic failure"), + (False, 1, "0.01001", "cost guardrail exhausted"), ]) def test_failure_or_exhaustion_stops_and_keeps_partial_evidence( workspace, monkeypatch, capsys, failure, calls, cost, reason, diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 8eb65ae..11900f8 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -41,7 +41,7 @@ def build_parser() -> argparse.ArgumentParser: run = commands.add_parser( "run", help="preview or execute an Evaluation run with one selected model", description=( - "Haiku 4.5 uses a 16,000-token thinking budget; Haiku 5.5, Opus 5.5 " + "Haiku 5.5 (default), Opus 5.5 " "and Sonnet 5.5 use adaptive thinking with medium effort. GPT models " "use Chat Completions with medium reasoning effort. All presets omit " "temperature, disable summaries, and cap output at 24,192 tokens " diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index 6feb828..a6d8b9d 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -15,15 +15,9 @@ from vision_aid.evaluation.normalization import normalize_prompt_response from vision_aid.evaluation.pricing import PriceSchedule -DEFAULT_MODEL = "claude-haiku-4-5-20251001" +DEFAULT_MODEL = "claude-haiku-5-5" OPENAI_MODELS = ("gpt-6-luna", "gpt-6.1-sol") MODEL_PRESETS = { - DEFAULT_MODEL: LLMRequestConfig( - model=DEFAULT_MODEL, - thinking_budget_tokens=16000, - max_output_tokens=24192, - temperature=None, - ), **{ model: LLMRequestConfig( model=model, @@ -32,7 +26,7 @@ max_output_tokens=24192, temperature=None, ) - for model in ("claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5") + for model in (DEFAULT_MODEL, "claude-opus-5-5", "claude-sonnet-5-5") }, **{ model: LLMRequestConfig( diff --git a/vision_aid/evaluation/pricing.json b/vision_aid/evaluation/pricing.json index 653c99c..170f8c4 100644 --- a/vision_aid/evaluation/pricing.json +++ b/vision_aid/evaluation/pricing.json @@ -1,11 +1,5 @@ { "models": { - "claude-haiku-4-5-20251001": { - "input_per_million_usd": "1", - "output_per_million_usd": "5", - "cached_input_per_million_usd": "0.10", - "cache_creation_input_per_million_usd": "1.25" - }, "claude-opus-5-5": { "input_per_million_usd": "4", "output_per_million_usd": "20", From 9836f033bd2595123ddc33ba6514809e85beeddd Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:45:23 -0400 Subject: [PATCH 074/126] docs(evaluation): record full evaluation domain and scoring rules --- CONTEXT.md | 12 ++++++++++-- .../0003-count-global-findings-once-across-pages.md | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0003-count-global-findings-once-across-pages.md diff --git a/CONTEXT.md b/CONTEXT.md index dcb9360..281e2d1 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,6 +1,6 @@ # Accessibility Audit Evaluation -This context names the evidence and measures for the homepage accessibility experiment. +This context names the evidence and measures for accessibility evaluations against the Human audit. ## Language @@ -12,6 +12,10 @@ _Avoid_: Reference workbook, workbook, ground-truth spreadsheet, generated revie One accessibility defect from the Human audit, kept with its original evidence and with the sheet and row it came from. Its `source_element` is the element name the testers wrote, not a precise inferred location. _Avoid_: Reference defect, workbook row, expected finding, inferred location. +**Global finding**: +A Human finding that applies to every page in the Human audit. In a Full evaluation, complete detection on any single page counts once toward the combined results. +_Avoid_: Homepage-only finding, separate Human finding for each page. + **Benchmark snapshot**: The saved HTML of the audited page, identified by its source URL and checksum. _Avoid_: Live page. @@ -20,6 +24,10 @@ _Avoid_: Live page. One audit with one selected model held fixed throughout, together with its evidence, the human decisions, its completion status, and its report. _Avoid_: Execution plan, report bundle. +**Full evaluation**: +An evaluation of every page listed in the Human audit with one selected model held fixed throughout. It includes results for each page and combined detection rates based on Human finding counts. +_Avoid_: Site crawl, model comparison. + **LLM**: The part of the tool that asks a large language model to judge the page for accessibility problems. _Avoid_: Audit model, AI audit. @@ -45,7 +53,7 @@ A reviewer's decision about which part of the tool should be expected to catch a _Avoid_: Automatic model score, exclusion without rationale. **Not testable from the page file**: -The reader-facing group for Human findings classified `unavailable_evidence` or `ambiguous`. They are left out of every detection rate, and each one is reported with its reason. +The reader-facing group for Human findings classified `unavailable_evidence` or `ambiguous`. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator, and each one is reported with its reason. _Avoid_: Excluded, unavailable, ambiguous (in reports). **Match decision**: diff --git a/docs/adr/0003-count-global-findings-once-across-pages.md b/docs/adr/0003-count-global-findings-once-across-pages.md new file mode 100644 index 0000000..52daf36 --- /dev/null +++ b/docs/adr/0003-count-global-findings-once-across-pages.md @@ -0,0 +1,3 @@ +# Count Global findings once across all pages + +Global findings apply to every page listed in the Human audit, even when their reproduction steps name the homepage. A Full evaluation awards combined credit for complete detection on any single page and counts each original Global finding once in the denominator. This measures whether the tool detects the reported defect anywhere in the evaluated pages; requiring detection on every page or multiplying the finding by page count would measure a different outcome. From ddd3b1de3bb7797d77dbae89f910fe999bbd1299 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:47:25 -0400 Subject: [PATCH 075/126] refactor(evaluation): share page preparation and reviewed reports --- tests/test_evaluation.py | 13 +++ vision_aid/evaluation/cli.py | 106 ++-------------------- vision_aid/evaluation/page.py | 151 ++++++++++++++++++++++++++++++++ vision_aid/evaluation/review.py | 48 ++++++++-- 4 files changed, 212 insertions(+), 106 deletions(-) create mode 100644 vision_aid/evaluation/page.py diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 37fe897..2437293 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -594,6 +594,19 @@ def test_historical_reports_keep_saved_prices_and_stored_estimate(workspace, mon assert "180000 input tokens" in text and "18000 output tokens" in text assert all(path.read_bytes() == saved for path, saved in originals.items()) + +def test_report_describes_the_evaluated_page(workspace, monkeypatch): + """A saved page's report identifies its URL without claiming homepage scope.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--source-url", "https://example.test/contact/", + "--live", "--approve-live") == 0 + run = only_run(workspace) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + text = report(run) + assert "This evaluation covers the saved page at https://example.test/contact/" in text + assert "homepage only" not in text + + def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): """Known decisions yield hand-calculated rates, cross-catches, and exclusions.""" fake_client( diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 11900f8..ce85591 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -3,20 +3,15 @@ from __future__ import annotations import argparse -import hashlib -import json import os -import shutil import sys import uuid from collections.abc import Sequence -from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path from dotenv import load_dotenv -from entry_points.run_pipeline import run_pipeline from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( DEFAULT_MODEL, @@ -24,11 +19,9 @@ evaluation_configuration, execute, positive_cost, - write_json, ) -from vision_aid.evaluation.normalization import normalize_programmatic_findings -from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule -from vision_aid.evaluation.review import create_review, reviewed_rows, write_report +from vision_aid.evaluation.page import prepare_page, report_page +from vision_aid.evaluation.pricing import PriceSchedule PROJECT_ROOT = Path(__file__).resolve().parents[2] DEFAULT_HUMAN_AUDIT = PROJECT_ROOT / "Pristine Accessibility Defect Report.xlsx" @@ -106,67 +99,11 @@ def _run(args: argparse.Namespace) -> int: raise ValueError(f"Benchmark HTML not found: {args.html}") human_audit = import_homepage_human_findings(args.human_audit, args.source_url) run = _new_run_directory() - artifacts = run / "artifacts" - artifacts.mkdir() - shutil.copyfile(args.html, run / "snapshot.html") - shutil.copyfile(DEFAULT_SCHEDULE, artifacts / "pricing.json") - snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() - write_json(artifacts / "human-findings.json", human_audit.to_dict()) - production = run_pipeline( - html_path=str(run / "snapshot.html"), - output_dir=artifacts, - api_key=None, - model=request_config.model, - dry_run=True, - include_summaries=False, - request_config=request_config, + manifest = prepare_page( + run, args.html, args.source_url, human_audit, request_config, limit ) - (artifacts / "manifest.json").unlink() - raw_path = artifacts / "programmatic_findings.json" - raw_programmatic = json.loads(raw_path.read_text()) - raw_path.rename(artifacts / "raw-programmatic-findings.json") - findings = normalize_programmatic_findings( - raw_programmatic, - run_id=run.name, - page_url=args.source_url, - ) - write_json( - run / "normalized-programmatic-findings.json", - [asdict(item) for item in findings], - ) - create_review(run / "review.csv", list(human_audit.to_dict()["human_findings"])) - schedule = PriceSchedule.load(artifacts / "pricing.json") - manifest = { - "run_id": run.name, - "source_url": args.source_url, - "human_audit_filename": human_audit.human_audit_filename, - "human_audit_sha256": human_audit.human_audit_sha256, - "snapshot_sha256": snapshot_hash, - "configuration": configuration, - "pricing_version": schedule.version, - "pricing_sha256": schedule.identity, - "max_cost_usd": str(limit), - "complete": False, - "incomplete_reason": "preview", - "estimated_cost_usd": "0", - "usage": { - key: 0 - for key in ( - "input_tokens", - "cached_input_tokens", - "cache_creation_input_tokens", - "output_tokens", - ) - }, - "format_failures": [], - "prompts": [ - {"name": entry["name"], "checklist": entry["checklist"]} - for entry in production["prompts_dry_run"] - ], - "filters": production["pass1_filters_active"], - "skipped_prompts": production["prompts_skipped"], - } - write_json(run / "run.json", manifest) + snapshot_hash = manifest["snapshot_sha256"] + schedule = PriceSchedule.load(run / "artifacts" / "pricing.json") print(f"Evaluation run: {run}") print( f"Model: {request_config.model}; provider: {configuration['provider']}; " @@ -233,37 +170,8 @@ def _run(args: argparse.Namespace) -> int: def _report(args: argparse.Namespace) -> int: """Validate human decisions and report a completed run.""" - if not args.reviewer.strip(): - raise ValueError("A reviewer description is required") run = args.run_dir.resolve() - artifacts = run / "artifacts" - manifest = json.loads((run / "run.json").read_text()) - if manifest["run_id"] != run.name: - raise ValueError("Run evidence does not belong to this directory") - if not manifest["complete"]: - raise ValueError("Incomplete Evaluation runs cannot produce a final report") - human_audit = json.loads((artifacts / "human-findings.json").read_text()) - if human_audit["human_audit_sha256"] != manifest["human_audit_sha256"]: - raise ValueError("Human audit identity differs from the run") - llm = json.loads((run / "normalized-llm-findings.json").read_text()) - programmatic_file = run / "normalized-programmatic-findings.json" - programmatic = json.loads(programmatic_file.read_text()) - for kind, collection in (("llm", llm), ("programmatic", programmatic)): - if any( - item["run_id"] != run.name - or item["source"] != kind - or item["page_url"] != manifest["source_url"] - for item in collection - ): - raise ValueError(f"{kind.upper() if kind == 'llm' else kind.title()} findings do not belong to this run") - rows = reviewed_rows( - run / "review.csv", human_audit["human_findings"], llm, programmatic - ) - schedule = PriceSchedule.load(artifacts / "pricing.json") - write_report( - run / "report.md", manifest, rows, llm, programmatic, - args.reviewer, schedule.price_description, - ) + report_page(run, args.reviewer) print(f"Markdown report: {run / 'report.md'}") return 0 diff --git a/vision_aid/evaluation/page.py b/vision_aid/evaluation/page.py new file mode 100644 index 0000000..69b1261 --- /dev/null +++ b/vision_aid/evaluation/page.py @@ -0,0 +1,151 @@ +"""Shared preparation and human-reviewed evidence for one evaluated page.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +from dataclasses import asdict, dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any + +from entry_points.run_pipeline import run_pipeline +from vision_aid.evaluation.llm_audit import ( + LLMRequestConfig, + USAGE_KEYS, + evaluation_configuration, + write_json, +) +from vision_aid.evaluation.normalization import normalize_programmatic_findings +from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule +from vision_aid.evaluation.review import create_review, reviewed_rows, write_report +from vision_aid.evaluation.schemas import HumanAudit + + +@dataclass(frozen=True) +class PageReview: + """A completed page's validated review and authoritative saved evidence.""" + + manifest: dict[str, Any] + rows: list[dict[str, Any]] + llm: list[dict[str, Any]] + programmatic: list[dict[str, Any]] + schedule: PriceSchedule + + +def prepare_page( + run: Path, + html: Path, + source_url: str, + human_audit: HumanAudit, + request_config: LLMRequestConfig, + limit: Decimal, +) -> dict[str, Any]: + """Save a page's inputs, checks, prompts and blank review without paid requests. + + The caller supplies an existing fresh run directory and applicable Human + findings, retaining the same operations for one-page and Full evaluations. + """ + if not html.is_file(): + raise ValueError(f"Benchmark HTML not found: {html}") + configuration = evaluation_configuration(request_config) + artifacts = run / "artifacts" + artifacts.mkdir() + shutil.copyfile(html, run / "snapshot.html") + shutil.copyfile(DEFAULT_SCHEDULE, artifacts / "pricing.json") + snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() + write_json(artifacts / "human-findings.json", human_audit.to_dict()) + production = run_pipeline( + html_path=str(run / "snapshot.html"), + output_dir=artifacts, + api_key=None, + model=request_config.model, + dry_run=True, + include_summaries=False, + request_config=request_config, + ) + (artifacts / "manifest.json").unlink() + raw_path = artifacts / "programmatic_findings.json" + raw_programmatic = json.loads(raw_path.read_text()) + raw_path.rename(artifacts / "raw-programmatic-findings.json") + findings = normalize_programmatic_findings( + raw_programmatic, + run_id=run.name, + page_url=source_url, + ) + write_json( + run / "normalized-programmatic-findings.json", + [asdict(item) for item in findings], + ) + create_review( + run / "review.csv", list(human_audit.to_dict()["human_findings"]) + ) + schedule = PriceSchedule.load(artifacts / "pricing.json") + manifest = { + "run_id": run.name, + "source_url": source_url, + "human_audit_filename": human_audit.human_audit_filename, + "human_audit_sha256": human_audit.human_audit_sha256, + "snapshot_sha256": snapshot_hash, + "configuration": configuration, + "pricing_version": schedule.version, + "pricing_sha256": schedule.identity, + "max_cost_usd": str(limit), + "complete": False, + "incomplete_reason": "preview", + "estimated_cost_usd": "0", + "usage": {key: 0 for key in USAGE_KEYS}, + "format_failures": [], + "prompts": [ + {"name": entry["name"], "checklist": entry["checklist"]} + for entry in production["prompts_dry_run"] + ], + "filters": production["pass1_filters_active"], + "skipped_prompts": production["prompts_skipped"], + } + write_json(run / "run.json", manifest) + return manifest + + +def validated_page_review(run: Path) -> PageReview: + """Load and validate a completed page's evidence and human decisions.""" + run = run.resolve() + artifacts = run / "artifacts" + manifest = json.loads((run / "run.json").read_text()) + if manifest["run_id"] != run.name: + raise ValueError("Run evidence does not belong to this directory") + if not manifest["complete"]: + raise ValueError("Incomplete Evaluation runs cannot produce a final report") + human_audit = json.loads((artifacts / "human-findings.json").read_text()) + if human_audit["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the run") + llm = json.loads((run / "normalized-llm-findings.json").read_text()) + programmatic_file = run / "normalized-programmatic-findings.json" + programmatic = json.loads(programmatic_file.read_text()) + for kind, collection in (("llm", llm), ("programmatic", programmatic)): + if any( + item["run_id"] != run.name + or item["source"] != kind + or item["page_url"] != manifest["source_url"] + for item in collection + ): + label = kind.upper() if kind == "llm" else kind.title() + raise ValueError(f"{label} findings do not belong to this run") + rows = reviewed_rows( + run / "review.csv", human_audit["human_findings"], llm, programmatic + ) + schedule = PriceSchedule.load(artifacts / "pricing.json") + return PageReview(manifest, rows, llm, programmatic, schedule) + + +def report_page(run: Path, reviewer: str) -> PageReview: + """Validate human decisions and write the page report with attribution.""" + if not reviewer.strip(): + raise ValueError("A reviewer description is required") + review = validated_page_review(run) + write_report( + run / "report.md", review.manifest, review.rows, review.llm, + review.programmatic, reviewer, review.schedule.price_description, + ) + return review diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 49cc805..e1569b7 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -6,6 +6,7 @@ from decimal import Decimal from pathlib import Path from typing import Any +from urllib.parse import urlsplit from vision_aid.evaluation.human_audit import ELIGIBILITY_VALUES, HEADERS @@ -130,6 +131,32 @@ def _rate(part: str, caught: int, total: int) -> str: return f"{label} detection rate: Of the {total} Human findings the {part} could be expected to detect from the page's HTML, {subject} caught {caught} ({caught / total:.1%})." +def detection_counts(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]: + """Count validated catches and denominators without averaging page rates. + + Each row represents one Human finding. Cross-catches earn Overall credit, + while source rates include only catches by the expected source. Not-testable + findings remain in the Overall denominator. + """ + counts = {} + for classification, kind in ( + ("llm_eligible", "llm"), ("programmatic", "programmatic") + ): + eligible = [ + row for row in rows + if row["decision"]["classification"].strip() == classification + ] + counts[kind] = { + "caught": sum(bool(row["matches"][kind]) for row in eligible), + "eligible": len(eligible), + } + counts["overall"] = { + "caught": sum(any(row["matches"].values()) for row in rows), + "eligible": len(rows), + } + return counts + + def _md(value: object) -> str: """Escape inline text for Markdown, including raw HTML, keeping line breaks.""" text = str(value) @@ -158,6 +185,12 @@ def write_report( "ambiguous": "Not testable", } covered = [row for row in rows if any(row["matches"].values())] + counts = detection_counts(rows) + page_scope = ( + "This evaluation covers the homepage only" + if urlsplit(manifest["source_url"]).path in {"", "/"} + else f"This evaluation covers the saved page at {_md(manifest['source_url'])}" + ) usage = manifest["usage"] tokens = [f"{usage['input_tokens']} input tokens", f"{usage['output_tokens']} output tokens"] for key, label in (("cached_input_tokens", "cached input"), ("cache_creation_input_tokens", "cache-creation input")): @@ -171,7 +204,7 @@ def write_report( "The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.", "This API uses LLM checks and Programmatic checks. LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.", "LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.", - "The tool saw only saved HTML, not a live browser or screen reader. This evaluation covers the homepage only, including Global findings that apply there.", + f"The tool saw only saved HTML, not a live browser or screen reader. {page_scope}, including Global findings that apply there.", "A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.", "", "## Summary", @@ -180,14 +213,15 @@ def write_report( f"Review decisions by: {_md(reviewer)}", "", ] - for classification, kind, part in ( - ("llm_eligible", "llm", "LLM"), - ("programmatic", "programmatic", "Programmatic checks"), + for kind, part in ( + ("llm", "LLM"), + ("programmatic", "Programmatic checks"), ): - eligible = [row for row in rows if row["decision"]["classification"].strip() == classification] - lines.extend([_rate(part, sum(bool(row["matches"][kind]) for row in eligible), len(eligible)), ""]) + lines.extend([ + _rate(part, counts[kind]["caught"], counts[kind]["eligible"]), "" + ]) lines.extend([ - _rate("Overall", len(covered), len(rows)), + _rate("Overall", counts["overall"]["caught"], counts["overall"]["eligible"]), "", f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated using provider-reported token usage and the {_md(pricing_description)}. Reported totals: {', '.join(tokens)}. The actual bill may differ slightly.", "", From d1deb2d9538c897d32c93fc5fecf93c1f192da3f Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:47:36 -0400 Subject: [PATCH 076/126] feat(evaluation): capture listed Benchmark snapshots --- README.md | 27 +++++- tests/test_evaluation_capture.py | 126 +++++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 18 +++- vision_aid/evaluation/full.py | 32 +++++++ vision_aid/evaluation/human_audit.py | 22 +++++ 5 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 tests/test_evaluation_capture.py create mode 100644 vision_aid/evaluation/full.py diff --git a/README.md b/README.md index 4f33794..74639a5 100644 --- a/README.md +++ b/README.md @@ -424,8 +424,8 @@ requests run sequentially without retries or fallback models. From the repository root, use the supplied `Pristine Accessibility Defect Report.xlsx` (16 Home and Global defects). These steps work after deleting `.model-evaluation/`. -1. **Save a Benchmark snapshot.** The evaluator does not download the page. - Create the ignored directory and save the homepage HTML there: +1. **Save a Benchmark snapshot.** For the one-page workflow, + create the ignored directory and save the homepage HTML there: ```bash mkdir -p .model-evaluation @@ -507,6 +507,29 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report Programmatic rates but remain in the Overall denominator. Reporting rejects incomplete runs, missing classifications, and invalid or reused finding IDs. +### Capture the Human audit's listed pages + +Save one Benchmark snapshot for each URL in the Human audit's `URL list` sheet: + +```bash +uv run visionaid-evaluate capture \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --output-dir .model-evaluation/pristine-snapshots +``` + +The output directory must be new. Capture downloads only the listed URLs in +source order and follows redirects, retaining each originally listed URL as +the page identity. It saves separate `page-001.html`, `page-002.html`, and later +HTML files, then writes `snapshots.json` with the Human audit checksum and +source URLs mapped to relative HTML filenames. Resolve those filenames beside +`snapshots.json` when reusing the saved inputs. + +Capture needs no model API key or Live-run approval. It saves server HTML and +does not render JavaScript or discover more pages. An HTTP or network error +stops capture immediately. Earlier HTML files remain, but capture writes no +snapshot mapping until every download succeeds. Start another capture in a +new directory after addressing the failure. + ### How the evaluator reads LLM responses The evaluator reads each LLM response using va-dat's own parsing and reporting diff --git a/tests/test_evaluation_capture.py b/tests/test_evaluation_capture.py new file mode 100644 index 0000000..7863b7d --- /dev/null +++ b/tests/test_evaluation_capture.py @@ -0,0 +1,126 @@ +"""Capture Benchmark snapshots through the public evaluator CLI.""" + +import hashlib +import json +from pathlib import Path + +import openpyxl +import pytest +import requests + +from vision_aid.evaluation.cli import main + + +def audit(path: Path, *urls: str | None) -> Path: + """Create a synthetic URL list without private Human audit evidence.""" + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "URL list" + sheet.append(["#", "Web page name", "URL", "Status"]) + for number, url in enumerate(urls, start=1): + sheet.append([number, "Repeated name", url, ""]) + book.save(path) + return path + + +def capture(human_audit: Path, output: Path) -> int: + return main(["capture", "--human-audit", str(human_audit), "--output-dir", str(output)]) + + +def test_capture_saves_only_listed_urls_in_order_and_preserves_redirect_identity( + tmp_path, monkeypatch, capsys, +): + """A key-free capture saves separate HTML and a reusable relative mapping.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/old/", None, + "https://example.test/contact/") + output = tmp_path / "snapshots" + downloads = [] + + def get(url, **options): + downloads.append((url, options)) + response = requests.Response() + response.status_code = 200 + response.url = "https://example.test/redirected/" + response._content = (b"First page" if len(downloads) == 1 + else b"Second page") + return response + + monkeypatch.setattr(requests, "get", get) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("provider access")) + assert capture(source, output) == 0 + mapping = json.loads((output / "snapshots.json").read_text()) + assert mapping == { + "human_audit_filename": "audit.xlsx", + "human_audit_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "snapshots": [ + {"source_url": "https://example.test/old/", "html": "page-001.html"}, + {"source_url": "https://example.test/contact/", "html": "page-002.html"}, + ], + } + assert (output / "page-001.html").read_text() == "First page" + assert (output / "page-002.html").read_text() == "Second page" + assert [url for url, _ in downloads] == [ + "https://example.test/old/", "https://example.test/contact/", + ] + assert all(options.get("allow_redirects", True) for _, options in downloads) + assert str(output / "snapshots.json") in capsys.readouterr().out + + +@pytest.mark.parametrize("status", [404, 503]) +def test_capture_stops_after_http_error_and_keeps_saved_evidence( + tmp_path, monkeypatch, capsys, status, +): + """Failed download leaves earlier HTML but no complete snapshot mapping.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/first/", + "https://example.test/failing/", "https://example.test/later/") + output = tmp_path / "snapshots" + downloads = [] + + def get(url, **options): + downloads.append(url) + response = requests.Response() + response.status_code = 200 if len(downloads) == 1 else status + response.url = url + response._content = b"Saved first page" + return response + + monkeypatch.setattr(requests, "get", get) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert downloads == ["https://example.test/first/", "https://example.test/failing/"] + assert {path.name for path in output.iterdir()} == {"page-001.html"} + assert (output / "page-001.html").read_text() == "Saved first page" + assert str(status) in capsys.readouterr().err + + +def test_capture_requires_a_new_directory_before_downloading(tmp_path, monkeypatch, capsys): + """Saved Benchmark snapshots cannot be replaced by another capture.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/") + output = tmp_path / "snapshots" + output.mkdir() + (output / "saved.html").write_text("Previously captured") + monkeypatch.setattr(requests, "get", lambda *_, **__: pytest.fail("download")) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert (output / "saved.html").read_text() == "Previously captured" + assert "exists" in capsys.readouterr().err.lower() + + +@pytest.mark.parametrize("urls", [(), (None, " ")]) +def test_capture_rejects_an_empty_url_list_before_downloading( + tmp_path, monkeypatch, capsys, urls, +): + """An audit without targets cannot publish a complete snapshot set.""" + source = audit(tmp_path / "audit.xlsx", *urls) + output = tmp_path / "snapshots" + monkeypatch.setattr(requests, "get", lambda *_, **__: pytest.fail("download")) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert not output.exists() + assert "no target URLs" in capsys.readouterr().err diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 11900f8..02788d1 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -17,6 +17,7 @@ from dotenv import load_dotenv from entry_points.run_pipeline import run_pipeline +from vision_aid.evaluation.full import capture_snapshots from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( DEFAULT_MODEL, @@ -35,9 +36,20 @@ def build_parser() -> argparse.ArgumentParser: - """Expose the two supported evaluation operations.""" + """Expose the supported evaluation operations.""" parser = argparse.ArgumentParser(prog="visionaid-evaluate") commands = parser.add_subparsers(dest="command", required=True) + capture = commands.add_parser( + "capture", help="download each Human audit URL to a saved Benchmark snapshot", + description=( + "Download listed URLs in source order, following redirects. " + "No model key or approval is required." + ), + ) + capture.add_argument("--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT) + capture.add_argument( + "--output-dir", type=Path, required=True, help="new snapshot directory" + ) run = commands.add_parser( "run", help="preview or execute an Evaluation run with one selected model", description=( @@ -273,6 +285,10 @@ def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) try: + if args.command == "capture": + mapping = capture_snapshots(args.human_audit, args.output_dir) + print(f"Snapshot mapping: {mapping}") + return 0 return _run(args) if args.command == "run" else _report(args) except (OSError, ValueError, KeyError) as error: parser.error(str(error)) diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py new file mode 100644 index 0000000..7b9f324 --- /dev/null +++ b/vision_aid/evaluation/full.py @@ -0,0 +1,32 @@ +"""Coordinate Full evaluation preparation and saved Benchmark snapshots.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import requests + +from vision_aid.evaluation.human_audit import list_audit_urls +from vision_aid.evaluation.llm_audit import write_json + + +def capture_snapshots(human_audit: Path, output_dir: Path) -> Path: + """Download only listed pages and publish their mapping after all succeed.""" + urls = list_audit_urls(human_audit) + audit_checksum = hashlib.sha256(human_audit.read_bytes()).hexdigest() + output_dir.mkdir(parents=True) + snapshots = [] + for number, source_url in enumerate(urls, start=1): + response = requests.get(source_url, allow_redirects=True) + response.raise_for_status() + html = f"page-{number:03d}.html" + (output_dir / html).write_bytes(response.content) + snapshots.append({"source_url": source_url, "html": html}) + mapping = output_dir / "snapshots.json" + write_json(mapping, { + "human_audit_filename": human_audit.name, + "human_audit_sha256": audit_checksum, + "snapshots": snapshots, + }) + return mapping diff --git a/vision_aid/evaluation/human_audit.py b/vision_aid/evaluation/human_audit.py index 05793ed..24adfcb 100644 --- a/vision_aid/evaluation/human_audit.py +++ b/vision_aid/evaluation/human_audit.py @@ -75,3 +75,25 @@ def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: if not human_findings: raise ValueError("Human audit has no Home or Global defects") return HumanAudit(path.name, checksum, homepage_url, tuple(human_findings)) + + +def list_audit_urls(path: Path) -> tuple[str, ...]: + """Read the Human audit's explicit target URLs in source order.""" + book = openpyxl.load_workbook(path, read_only=True, data_only=True) + try: + sheet = book["URL list"] + rows = sheet.iter_rows(values_only=True) + headers = tuple(str(value or "").strip() for value in next(rows)) + if "URL" not in headers: + raise ValueError("Human audit URL list has no URL column") + index = headers.index("URL") + urls = tuple( + str(row[index]).strip() + for row in rows + if len(row) > index and row[index] is not None and str(row[index]).strip() + ) + finally: + book.close() + if not urls: + raise ValueError("Human audit URL list has no target URLs") + return urls From c870fc350c28fdaef096b0bdfbac618b6c9d2d62 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:54:32 -0400 Subject: [PATCH 077/126] feat(evaluation): preview all Human audit pages from saved snapshots --- README.md | 34 ++++ tests/test_full_preview.py | 265 +++++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 118 +++++++++--- vision_aid/evaluation/full.py | 124 ++++++++++++- vision_aid/evaluation/human_audit.py | 63 ++++++- vision_aid/evaluation/schemas.py | 4 +- 6 files changed, 571 insertions(+), 37 deletions(-) create mode 100644 tests/test_full_preview.py diff --git a/README.md b/README.md index 74639a5..63e8659 100644 --- a/README.md +++ b/README.md @@ -530,6 +530,40 @@ stops capture immediately. Earlier HTML files remain, but capture writes no snapshot mapping until every download succeeds. Start another capture in a new directory after addressing the failure. +### Preview a Full evaluation + +Prepare every listed page from the saved Benchmark snapshots: + +```bash +uv run visionaid-evaluate run-all \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ + --model claude-haiku-5-5 \ + --max-cost-usd 5 +``` + +The snapshot mapping must match the supplied Human audit's checksum and every +listed URL in source order. HTML filenames resolve beside the mapping. The +preview uses saved HTML without downloading again and needs no model API key. +It displays all pages, the total prompt count, one fixed model preset, saved +token prices, and one positive allowance shared across the Full evaluation. + +Local Human findings map to pages by the listed target URL in Steps to +Reproduce, preserving original text and identities. Page names and URLs in +recommendations do not assign findings. The known `/Contact/` alias maps to +`/contact/`; other paths retain their case. Missing or ambiguous assignments +stop preparation. Each page's review contains its local findings and every +Global finding, including pages with no local findings. + +Each invocation creates a fresh `.model-evaluation/full-runs//` +directory. Its `full.json` records ordered membership and preparation status; +`human-findings.json` retains unique originals and their page assignments. +Each child under `pages/` keeps the existing Evaluation run files, including +its copied snapshot, prompts, pricing, Programmatic findings and review CSV. +Preview leaves execution incomplete and makes no paid requests. A preparation +failure stops later pages and retains earlier and partial evidence with its +stopping reason. + ### How the evaluator reads LLM responses The evaluator reads each LLM response using va-dat's own parsing and reporting diff --git a/tests/test_full_preview.py b/tests/test_full_preview.py new file mode 100644 index 0000000..d53ab34 --- /dev/null +++ b/tests/test_full_preview.py @@ -0,0 +1,265 @@ +"""Preview Full evaluations through the public evaluator CLI.""" + +import csv +import hashlib +import json +from pathlib import Path + +import openpyxl +import pytest + +from vision_aid.evaluation.cli import main +from vision_aid.evaluation.human_audit import HEADERS + +URLS = ["https://example.test/", "https://example.test/contact/", + "https://example.test/empty/"] +HTML = 'Test' + + +@pytest.fixture +def full_workspace(tmp_path, monkeypatch): + """Save a synthetic Human audit and reusable captured snapshot mapping.""" + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "Defect report" + sheet.append(HEADERS) + for number, page_name, title, steps in ( + (1, "Repeated name", "First local", f"Open {URLS[0]}"), + (2, "Repeated name", "Second local", f"Open {URLS[0]}"), + (3, "Home", "Contact local", "Open https://example.test/Contact/."), + (4, "Global", "Shared defect", f"Open {URLS[0]}"), + (5, "Global", "Shared second defect", f"Open {URLS[0]}"), + ): + sheet.append([number, "Element", "Chrome", page_name, title, steps, + "Original\nevidence", "Expected", f"Read {URLS[2]}", "1.1.1"]) + urls = book.create_sheet("URL list") + urls.append(["Page name", "URL"]) + for url in URLS: + urls.append(["Repeated name", url]) + book.save(tmp_path / "human-audit.xlsx") + saved = tmp_path / "snapshots" + saved.mkdir() + snapshots = [] + for number, url in enumerate(URLS, start=1): + filename = f"page-{number:03d}.html" + (saved / filename).write_text(HTML) + snapshots.append({"source_url": url, "html": filename}) + mapping = { + "human_audit_filename": "human-audit.xlsx", + "human_audit_sha256": hashlib.sha256((tmp_path / "human-audit.xlsx").read_bytes()).hexdigest(), + "snapshots": snapshots, + } + (saved / "snapshots.json").write_text(json.dumps(mapping)) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("model request")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("model request")) + monkeypatch.setattr("requests.get", lambda *_, **__: pytest.fail("HTTP download")) + return tmp_path + + +def preview(workspace, *extra): + """Invoke the public Full evaluation preview using saved evidence.""" + return main([ + "run-all", "--human-audit", str(workspace / "human-audit.xlsx"), + "--snapshots", str(workspace / "snapshots/snapshots.json"), + "--max-cost-usd", "2", *extra, + ]) + + +def only_full(workspace): + """Read the only saved Full evaluation manifest.""" + (run,) = (workspace / ".model-evaluation/full-runs").iterdir() + return run, json.loads((run / "full.json").read_text()) + + +def test_preview_maps_original_findings_and_keeps_page_evidence(full_workspace, capsys): + """URL assignment, Globals and separate saved evidence survive key-free preview.""" + assert preview(full_workspace) == 0 + run, manifest = only_full(full_workspace) + assert [page["source_url"] for page in manifest["pages"]] == URLS + assert len({page["run_id"] for page in manifest["pages"]}) == 3 + assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" + assert manifest["max_cost_usd"] == "2" and manifest["estimated_cost_usd"] == "0" + assert all(value == 0 for value in manifest["usage"].values()) + memberships = [] + globals_by_page = [] + programmatic_ids = set() + prompt_count = 0 + for page in manifest["pages"]: + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + assert child_manifest["run_id"] == child.name == page["run_id"] + assert child_manifest["source_url"] == page["source_url"] + assert child_manifest["configuration"] == manifest["configuration"] + assert (child / "snapshot.html").read_text() == HTML + assert child_manifest["snapshot_sha256"] == hashlib.sha256(HTML.encode()).hexdigest() + prompt_count += len(child_manifest["prompts"]) + with (child / "review.csv").open(newline="") as stream: + rows = list(csv.DictReader(stream)) + memberships.append([row["Issue Title"] for row in rows]) + globals_by_page.append({row["human_finding_id"] for row in rows if row["page_scope"] == "global"}) + assert all(row["actual result"] == "Original\nevidence" for row in rows) + assert all(not row["classification"] for row in rows) + findings = json.loads((child / "normalized-programmatic-findings.json").read_text()) + assert findings and all(item["page_url"] == page["source_url"] for item in findings) + assert not programmatic_ids.intersection(item["finding_id"] for item in findings) + programmatic_ids.update(item["finding_id"] for item in findings) + assert memberships == [ + ["First local", "Second local", "Shared defect", "Shared second defect"], + ["Contact local", "Shared defect", "Shared second defect"], + ["Shared defect", "Shared second defect"], + ] + assert globals_by_page[0] == globals_by_page[1] == globals_by_page[2] + originals = json.loads((run / "human-findings.json").read_text()) + assert len(originals["human_findings"]) == 5 + assert len(originals["page_assignments"]) == 3 + output = capsys.readouterr().out + assert all(url in output for url in URLS) + assert f"prompts: {prompt_count}" in output + assert "Cost guardrail: $2" in output and "Pricing:" in output + assert "Preview complete; no paid requests made." in output + + +def change_audit(workspace, steps): + """Replace a reproduction URL and refresh its captured identity.""" + path = workspace / "human-audit.xlsx" + book = openpyxl.load_workbook(path) + book["Defect report"].cell(2, 6, steps) + book.save(path) + mapping_path = workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + mapping_path.write_text(json.dumps(mapping)) + + +@pytest.mark.parametrize("steps", [ + "No reproduction URL", "Open https://example.test/EMPTY/", + f"Open {URLS[0]} and {URLS[1]}", +]) +def test_invalid_page_assignments_stop_before_preparation(full_workspace, steps, capsys): + """Missing, arbitrary case aliases and ambiguous targets cannot start a run.""" + change_audit(full_workspace, steps) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert "exactly one listed target URL" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize("problem", [ + "checksum", "missing", "duplicate", "reordered", "outside", "missing_html", +]) +def test_snapshot_mapping_must_match_the_human_audit(full_workspace, problem, capsys): + """Mismatched capture identities and page sets never prepare an evaluation.""" + path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(path.read_text()) + if problem == "checksum": + mapping["human_audit_sha256"] = "other audit" + elif problem == "missing": + mapping["snapshots"].pop() + elif problem == "duplicate": + mapping["snapshots"].append(mapping["snapshots"][0]) + elif problem == "reordered": + mapping["snapshots"].reverse() + elif problem == "outside": + mapping["snapshots"][0]["html"] = "../human-audit.xlsx" + else: + (full_workspace / "snapshots/page-001.html").unlink() + path.write_text(json.dumps(mapping)) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert not (full_workspace / ".model-evaluation").exists() + assert "Snapshot mapping" in capsys.readouterr().err + + +def test_page_preparation_failure_retains_parent_and_earlier_evidence(full_workspace, capsys): + """A failed HTML read stops preparation and records every unfinished member.""" + (full_workspace / "snapshots/page-002.html").write_bytes(b"\xff") + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + run, manifest = only_full(full_workspace) + assert manifest["complete"] is False + assert "preparation failed" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "prepared", "preparation_failed", "unprepared", + ] + first, second, third = [run / page["run_dir"] for page in manifest["pages"]] + assert (first / "review.csv").is_file() + assert (first / "artifacts/raw-programmatic-findings.json").is_file() + assert (second / "snapshot.html").read_bytes() == b"\xff" + assert not third.exists() + assert str(run) in capsys.readouterr().out + + +def test_audit_without_human_findings_cannot_start_evaluation(full_workspace, capsys): + """Listed pages alone provide no Human findings to score.""" + path = full_workspace / "human-audit.xlsx" + book = openpyxl.load_workbook(path) + book["Defect report"].delete_rows(2, 5) + book.save(path) + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + mapping_path.write_text(json.dumps(mapping)) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert "no Human findings" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize("model", [ + "claude-haiku-5-5", "claude-opus-5-5", "claude-sonnet-5-5", + "gpt-6-luna", "gpt-6.1-sol", +]) +def test_preview_displays_one_fixed_model_and_recorded_prices(full_workspace, capsys, model): + """Every page records the selected preset without provider credentials.""" + assert preview(full_workspace, "--model", model) == 0 + run, manifest = only_full(full_workspace) + config = manifest["configuration"] + assert config["model"] == model + assert config["reasoning_effort"] == "medium" and config["max_output_tokens"] == 24192 + assert config["temperature"] == "omitted" and config["summaries"] == "disabled" + for page in manifest["pages"]: + child = run / page["run_dir"] + saved = json.loads((child / "run.json").read_text()) + assert saved["configuration"] == config + prices = json.loads((child / "artifacts/pricing.json").read_text()) + assert model in prices["models"] + output = capsys.readouterr().out + provider = "openai" if model.startswith("gpt-") else "anthropic" + assert f"Model: {model}; provider: {provider}" in output + assert "input $" in output and "output $" in output and "cache read $" in output + + +def test_one_page_selection_and_human_ids_remain_compatible(full_workspace): + """The one-page command still selects Home and Globals with identical IDs.""" + assert preview(full_workspace) == 0 + full_run, _ = only_full(full_workspace) + originals = json.loads((full_run / "human-findings.json").read_text()) + assert main([ + "run", "--human-audit", str(full_workspace / "human-audit.xlsx"), + "--html", str(full_workspace / "snapshots/page-001.html"), + "--source-url", URLS[0], "--max-cost-usd", "2", + ]) == 0 + (page_run,) = (full_workspace / ".model-evaluation/runs").iterdir() + evidence = json.loads((page_run / "artifacts/human-findings.json").read_text()) + assert [finding["problem"] for finding in evidence["human_findings"]] == [ + "Contact local", "Shared defect", "Shared second defect", + ] + assert evidence["human_findings"] == originals["human_findings"][2:] + + +def test_run_all_help_explains_saved_snapshots_and_shared_allowance(capsys): + """Operators can discover the reusable inputs and key-free preview.""" + with pytest.raises(SystemExit) as error: + main(["run-all", "--help"]) + assert error.value.code == 0 + text = capsys.readouterr().out + assert "snapshots.json" in text and "shared across pages" in text + assert "Previews need no key" in text and "Haiku 5.5 (default)" in text + assert "24,192" in text and "medium" in text diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 928995e..b76567f 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -3,19 +3,25 @@ from __future__ import annotations import argparse +import json import os import sys import uuid from collections.abc import Sequence from datetime import UTC, datetime +from decimal import Decimal from pathlib import Path +from typing import Any from dotenv import load_dotenv -from vision_aid.evaluation.full import capture_snapshots +from vision_aid.evaluation.full import ( + capture_snapshots, load_full_inputs, prepare_full_evaluation, +) from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( DEFAULT_MODEL, + LLMRequestConfig, MODEL_PRESETS, evaluation_configuration, execute, @@ -73,6 +79,25 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="explicitly approve billable execution without a prompt", ) + run_all = commands.add_parser( + "run-all", help="preview every listed page with one shared cost allowance", + description=run.description, + ) + run_all.add_argument( + "--model", choices=MODEL_PRESETS, default=DEFAULT_MODEL, + help=f"fixed model preset (default: {DEFAULT_MODEL})", + ) + run_all.add_argument( + "--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT, help="Human audit" + ) + run_all.add_argument( + "--snapshots", type=Path, required=True, + help="saved snapshots.json mapping from capture; resolves HTML beside it", + ) + run_all.add_argument( + "--max-cost-usd", required=True, + help="positive total allowance shared across pages; checked between requests", + ) report = commands.add_parser( "report", help="score completed run and edited review.csv" ) @@ -81,7 +106,7 @@ def build_parser() -> argparse.ArgumentParser: return parser -def _new_run_directory() -> Path: +def _new_run_directory(kind: str = "runs") -> Path: """Create a unique private directory under the containing worktree root.""" current = Path.cwd() worktree = next( @@ -92,7 +117,7 @@ def _new_run_directory() -> Path: ), current, ) - root = worktree / ".model-evaluation" / "runs" + root = worktree / ".model-evaluation" / kind root.mkdir(parents=True, exist_ok=True) run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8] run = root / run_id @@ -117,6 +142,43 @@ def _run(args: argparse.Namespace) -> int: snapshot_hash = manifest["snapshot_sha256"] schedule = PriceSchedule.load(run / "artifacts" / "pricing.json") print(f"Evaluation run: {run}") + _display_model( + request_config, configuration, schedule, len(manifest["prompts"]), limit + ) + print( + f"Snapshot SHA-256: {snapshot_hash}; " + f"Human findings: {len(human_audit.human_findings)}" + ) + print(f"Review CSV: {run / 'review.csv'}") + if not args.live: + print("Preview complete; no paid requests made.") + return 0 + load_dotenv(PROJECT_ROOT / ".env") + key_name = ( + "OPENAI_API_KEY" if configuration["provider"] == "openai" else "ANTHROPIC_API_KEY" + ) + api_key = os.getenv(key_name) + if not api_key: + raise ValueError(f"Live execution requires {key_name}") + if not args.approve_live and ( + not sys.stdin.isatty() + or input("Type 'yes' to approve paid execution: ") != "yes" + ): + print("Live execution declined; no paid requests made.") + return 0 + complete = execute(run, manifest, api_key, request_config) + print(f"Run {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 + + +def _display_model( + request_config: LLMRequestConfig, + configuration: dict[str, Any], + schedule: PriceSchedule, + prompt_count: int, + limit: Decimal, +) -> None: + """Display fixed request settings, recorded prices and the shared guardrail.""" print( f"Model: {request_config.model}; provider: {configuration['provider']}; " f"endpoint: {configuration['endpoint']}; " @@ -127,11 +189,7 @@ def _run(args: argparse.Namespace) -> int: ) print( "Temperature: omitted; summaries: disabled; " - f"prompts: {len(manifest['prompts'])}" - ) - print( - f"Snapshot SHA-256: {snapshot_hash}; " - f"Human findings: {len(human_audit.human_findings)}" + f"prompts: {prompt_count}" ) print(f"Pricing: {schedule.version} ({schedule.identity})") rates = schedule.data["models"][request_config.model] @@ -158,26 +216,30 @@ def _run(args: argparse.Namespace) -> int: f"Cost guardrail: ${limit}; checked before each request; " "a request underway can exceed the limit" ) - print(f"Review CSV: {run / 'review.csv'}") - if not args.live: - print("Preview complete; no paid requests made.") - return 0 - load_dotenv(PROJECT_ROOT / ".env") - key_name = ( - "OPENAI_API_KEY" if configuration["provider"] == "openai" else "ANTHROPIC_API_KEY" + + +def _run_all(args: argparse.Namespace) -> int: + """Preview all listed pages using saved snapshots and no provider access.""" + request_config = MODEL_PRESETS[args.model] + limit = positive_cost(args.max_cost_usd) + audits, snapshots = load_full_inputs(args.human_audit, args.snapshots) + run = _new_run_directory("full-runs") + print(f"Full evaluation: {run}") + manifest = prepare_full_evaluation(run, audits, snapshots, request_config, limit) + prompt_count = 0 + for page in manifest["pages"]: + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + prompt_count += len(child_manifest["prompts"]) + print(f"Page: {page['source_url']}; Evaluation run: {child}") + print(f"Review CSV: {child / 'review.csv'}") + first_child = run / manifest["pages"][0]["run_dir"] + schedule = PriceSchedule.load(first_child / "artifacts/pricing.json") + _display_model( + request_config, manifest["configuration"], schedule, prompt_count, limit ) - api_key = os.getenv(key_name) - if not api_key: - raise ValueError(f"Live execution requires {key_name}") - if not args.approve_live and ( - not sys.stdin.isatty() - or input("Type 'yes' to approve paid execution: ") != "yes" - ): - print("Live execution declined; no paid requests made.") - return 0 - complete = execute(run, manifest, api_key, request_config) - print(f"Run {'complete' if complete else 'incomplete'}: {run}") - return 0 if complete else 1 + print("Preview complete; no paid requests made.") + return 0 def _report(args: argparse.Namespace) -> int: @@ -197,6 +259,8 @@ def main(argv: Sequence[str] | None = None) -> int: mapping = capture_snapshots(args.human_audit, args.output_dir) print(f"Snapshot mapping: {mapping}") return 0 + if args.command == "run-all": + return _run_all(args) return _run(args) if args.command == "run" else _report(args) except (OSError, ValueError, KeyError) as error: parser.error(str(error)) diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py index 7b9f324..7e21042 100644 --- a/vision_aid/evaluation/full.py +++ b/vision_aid/evaluation/full.py @@ -3,12 +3,20 @@ from __future__ import annotations import hashlib +import json +import uuid +from decimal import Decimal from pathlib import Path +from typing import Any import requests -from vision_aid.evaluation.human_audit import list_audit_urls -from vision_aid.evaluation.llm_audit import write_json +from vision_aid.evaluation.human_audit import import_all_human_findings, list_audit_urls +from vision_aid.evaluation.llm_audit import ( + LLMRequestConfig, USAGE_KEYS, evaluation_configuration, write_json, +) +from vision_aid.evaluation.page import prepare_page +from vision_aid.evaluation.schemas import HumanAudit def capture_snapshots(human_audit: Path, output_dir: Path) -> Path: @@ -30,3 +38,115 @@ def capture_snapshots(human_audit: Path, output_dir: Path) -> Path: "snapshots": snapshots, }) return mapping + + +def load_full_inputs( + human_audit: Path, mapping_path: Path, +) -> tuple[dict[str, HumanAudit], dict[str, Path]]: + """Load listed page assignments and saved snapshots without downloading.""" + audits = import_all_human_findings(human_audit) + mapping = json.loads(mapping_path.read_text(encoding="utf-8")) + checksum = next(iter(audits.values())).human_audit_sha256 + if not isinstance(mapping, dict) or mapping.get("human_audit_sha256") != checksum: + raise ValueError( + "Snapshot mapping Human audit identity differs from the supplied audit" + ) + entries = mapping.get("snapshots") + if not isinstance(entries, list) or any( + not isinstance(entry, dict) + or not isinstance(entry.get("source_url"), str) + or not isinstance(entry.get("html"), str) + for entry in entries + ): + raise ValueError( + "Snapshot mapping must contain source URLs and relative HTML filenames" + ) + if [entry["source_url"] for entry in entries] != list(audits): + raise ValueError( + "Snapshot mapping must match every listed target once in source order" + ) + snapshots = {} + directory = mapping_path.parent.resolve() + for entry in entries: + filename = Path(entry["html"]) + html = (directory / filename).resolve() + if filename.is_absolute() or not html.is_relative_to(directory): + raise ValueError( + "Snapshot mapping HTML files must be relative files beside the mapping" + ) + if not html.is_file(): + raise ValueError(f"Snapshot mapping Benchmark HTML not found: {filename}") + snapshots[entry["source_url"]] = html + return audits, snapshots + + +def prepare_full_evaluation( + run: Path, + audits: dict[str, HumanAudit], + snapshots: dict[str, Path], + request_config: LLMRequestConfig, + limit: Decimal, +) -> dict[str, Any]: + """Prepare all listed pages, retaining membership and progress on failure.""" + first_audit = next(iter(audits.values())) + pages = [] + originals = {} + assignments = {} + for number, (source_url, audit) in enumerate(audits.items(), start=1): + child_id = f"{run.name}-page-{number:03d}-{uuid.uuid4().hex[:8]}" + human_ids = [] + for finding in audit.to_dict()["human_findings"]: + identity = finding["human_finding_id"] + originals[identity] = finding + human_ids.append(identity) + if finding["page_scope"] != "global": + assignments[identity] = source_url + pages.append({ + "source_url": source_url, + "run_id": child_id, + "run_dir": f"pages/{child_id}", + "human_finding_ids": human_ids, + "status": "unprepared", + "complete": False, + }) + manifest = { + "run_id": run.name, + "human_audit_filename": first_audit.human_audit_filename, + "human_audit_sha256": first_audit.human_audit_sha256, + "configuration": evaluation_configuration(request_config), + "max_cost_usd": str(limit), + "estimated_cost_usd": "0", + "usage": {key: 0 for key in USAGE_KEYS}, + "complete": False, + "incomplete_reason": "preparing pages", + "pages": pages, + } + write_json(run / "human-findings.json", { + "human_audit_filename": first_audit.human_audit_filename, + "human_audit_sha256": first_audit.human_audit_sha256, + "human_findings": sorted(originals.values(), key=lambda item: item["source_row"]), + "page_assignments": assignments, + }) + write_json(run / "full.json", manifest) + for page in pages: + child = run / page["run_dir"] + try: + child.mkdir(parents=True) + child_manifest = prepare_page( + child, snapshots[page["source_url"]], page["source_url"], + audits[page["source_url"]], request_config, limit, + ) + except (OSError, ValueError, KeyError) as error: + page["status"] = "preparation_failed" + manifest["incomplete_reason"] = ( + f"page preparation failed for {page['source_url']}: {error}" + ) + write_json(run / "full.json", manifest) + raise + page.update( + status="prepared", snapshot_sha256=child_manifest["snapshot_sha256"] + ) + write_json(run / "full.json", manifest) + manifest["incomplete_reason"] = "preview" + write_json(run / "full.json", manifest) + return manifest diff --git a/vision_aid/evaluation/human_audit.py b/vision_aid/evaluation/human_audit.py index 24adfcb..7f86b31 100644 --- a/vision_aid/evaluation/human_audit.py +++ b/vision_aid/evaluation/human_audit.py @@ -1,10 +1,11 @@ -"""Import the known Human audit's Home and Global defect rows.""" +"""Preserve Human findings and select them for one page or every listed page.""" from __future__ import annotations import hashlib import re from pathlib import Path +from urllib.parse import urlsplit import openpyxl @@ -33,8 +34,10 @@ ) -def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: - """Preserve every applicable source row and its original cell text.""" +def _source_findings( + path: Path, *, homepage_only: bool +) -> tuple[str, tuple[HumanFinding, ...]]: + """Read original Human finding evidence with stable source identities.""" checksum = hashlib.sha256(path.read_bytes()).hexdigest() book = openpyxl.load_workbook(path, read_only=True, data_only=True) try: @@ -51,7 +54,7 @@ def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: if header } scope = raw["Page name"].strip().casefold() - if scope not in {"home", "global"}: + if homepage_only and scope not in {"home", "global"}: continue if not raw["Issue Title"].strip(): raise ValueError(f"Defect report row {number} has no Issue Title") @@ -61,7 +64,7 @@ def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: human_finding_id="human-" + hashlib.sha256(identity).hexdigest()[:16], source_sheet=SHEET, source_row=number, - page_scope=scope, + page_scope=scope if scope in {"home", "global"} else "page", source_element=raw["element name"], problem=raw["Issue Title"], raw_evidence=raw, @@ -72,9 +75,57 @@ def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: ) finally: book.close() + return checksum, tuple(human_findings) + + +def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: + """Preserve the one-page command's Home and Global source selection.""" + checksum, human_findings = _source_findings(path, homepage_only=True) if not human_findings: raise ValueError("Human audit has no Home or Global defects") - return HumanAudit(path.name, checksum, homepage_url, tuple(human_findings)) + return HumanAudit(path.name, checksum, homepage_url, human_findings) + + +def import_all_human_findings(path: Path) -> dict[str, HumanAudit]: + """Assign local findings by reproduction URL and repeat Globals per page.""" + urls = list_audit_urls(path) + if len(urls) != len(set(urls)): + raise ValueError("Human audit URL list contains repeated target URLs") + checksum, findings = _source_findings(path, homepage_only=False) + if not findings: + raise ValueError("Human audit has no Human findings") + by_page: dict[str, list[HumanFinding]] = {url: [] for url in urls} + for finding in findings: + if finding.page_scope == "global": + for page_findings in by_page.values(): + page_findings.append(finding) + continue + steps = finding.raw_evidence["Steps to Reproduce"] + reproduced = { + match.rstrip(".,;:)]}") + for match in re.findall(r'https?://[^\s<>"\']+', steps) + } + assigned = set() + for url in reproduced: + if url in by_page: + assigned.add(url) + continue + parsed = urlsplit(url) + # This is the audit's known alias, not general path case folding. + if parsed.path == "/Contact/": + url = parsed._replace(path="/contact/").geturl() + if url in by_page: + assigned.add(url) + if len(assigned) != 1: + raise ValueError( + f"Defect report row {finding.source_row} must map to exactly " + "one listed target URL in Steps to Reproduce" + ) + by_page[assigned.pop()].append(finding) + return { + url: HumanAudit(path.name, checksum, url, tuple(page_findings)) + for url, page_findings in by_page.items() + } def list_audit_urls(path: Path) -> tuple[str, ...]: diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py index f46de59..646a9af 100644 --- a/vision_aid/evaluation/schemas.py +++ b/vision_aid/evaluation/schemas.py @@ -1,4 +1,4 @@ -"""Evidence records for the fixed homepage evaluation.""" +"""Evidence records for page evaluations against the Human audit.""" from __future__ import annotations @@ -11,7 +11,7 @@ class HumanFinding: human_finding_id: str source_sheet: str source_row: int - page_scope: Literal["home", "global"] + page_scope: Literal["home", "global", "page"] source_element: str problem: str raw_evidence: dict[str, str] From e8429a54bcb44092e4a08fb1599269ab00a891c3 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 16:59:10 -0400 Subject: [PATCH 078/126] feat(evaluation): execute all pages with one cost allowance --- README.md | 39 +++++ tests/test_full_execution.py | 272 ++++++++++++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 47 ++++-- vision_aid/evaluation/full.py | 50 ++++++- 4 files changed, 396 insertions(+), 12 deletions(-) create mode 100644 tests/test_full_execution.py diff --git a/README.md b/README.md index 63e8659..0a4c397 100644 --- a/README.md +++ b/README.md @@ -564,6 +564,45 @@ Preview leaves execution incomplete and makes no paid requests. A preparation failure stops later pages and retains earlier and partial evidence with its stopping reason. +### Execute a Full evaluation + +Add `--live` to the preview command to execute all prepared pages with the +selected provider's API key: + +```bash +uv run visionaid-evaluate run-all \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ + --model claude-haiku-5-5 \ + --max-cost-usd 5 \ + --live +``` + +Every page is prepared before the CLI displays the full scope and asks for +one Live-run approval. Type `yes` to approve all listed pages, or add +`--approve-live` for explicit noninteractive approval. That option requires +`--live`. Declined approval makes no model requests. GPT models require +`OPENAI_API_KEY`; Claude models require `ANTHROPIC_API_KEY`. + +Pages execute in URL-list order, using the same model preset throughout. +The allowance covers the entire Full evaluation. Before each page, the CLI +subtracts prior reported estimated cost and passes the remainder to that +page's executor, which checks the limit before each request. A request underway +can exceed the limit. If it completes all remaining work, execution still +completes; otherwise the CLI starts no further requests once the allowance +is exhausted. Request failures also stop later work, without retries. + +The parent `full.json` saves page completion, stopping reason, estimated cost, +and reported usage, including usage from failed requests and partial pages. +Completed and partial child evidence remain available. A completed execution +still needs human review before final scoring. Completed page runs can use +the existing `report` command even if a later page fails. + +Start a fresh invocation after failure. Each invocation creates new parent +and child directories and repeats every model request, including pages that +completed in a previous attempt. It reuses captured HTML without downloading +again; it does not resume or reuse previous paid outputs. + ### How the evaluator reads LLM responses The evaluator reads each LLM response using va-dat's own parsing and reporting diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py new file mode 100644 index 0000000..d0bf469 --- /dev/null +++ b/tests/test_full_execution.py @@ -0,0 +1,272 @@ +"""Execute Full evaluations through the CLI with controlled provider replies.""" + +import json + +import pytest + +from test_evaluation import recording_provider_usage +from test_full_preview import URLS, full_workspace, only_full, preview + + +def children(run, manifest): + return [(run / page["run_dir"], json.loads((run / page["run_dir"] / "run.json").read_text())) + for page in manifest["pages"]] + + +def test_declined_full_approval_makes_no_requests_after_showing_all_pages( + full_workspace, monkeypatch, capsys, +): + """One refusal covers the whole prepared scope, even with credentials present.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + approvals = [] + + def decline(prompt): + approvals.append(prompt) + run, manifest = only_full(full_workspace) + assert all((child / "review.csv").exists() for child, _ in children(run, manifest)) + displayed = capsys.readouterr().out + assert all(url in displayed for url in URLS) + assert "Pricing:" in displayed and "Cost guardrail: $2" in displayed + return "no" + + monkeypatch.setattr("builtins.input", decline) + assert preview(full_workspace, "--live") == 0 + assert len(approvals) == 1 + _, manifest = only_full(full_workspace) + assert not manifest["complete"] and manifest["estimated_cost_usd"] == "0" + assert "declined" in capsys.readouterr().out + + +@pytest.mark.parametrize("model", ["claude-haiku-5-5", "gpt-6-luna"]) +def test_one_approval_executes_ordered_pages_with_one_fixed_preset( + full_workspace, monkeypatch, capsys, model, +): + """Each child's requests use the approved model and evidence belongs to its page.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + requests = recording_provider_usage(monkeypatch, model) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + approvals = [] + + def approve(prompt): + assert not requests + approvals.append(prompt) + run, manifest = only_full(full_workspace) + assert all((child / "snapshot.html").exists() for child, _ in children(run, manifest)) + return "yes" + + monkeypatch.setattr("builtins.input", approve) + assert preview(full_workspace, "--live", "--model", model) == 0 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert len(approvals) == 1 and manifest["complete"] + assert manifest["incomplete_reason"] is None + assert len(requests) == sum(len(child["prompts"]) for _, child in saved) > 0 + assert all(request["model"] == model for request in requests) + identities = set() + for child, child_manifest in saved: + assert child_manifest["complete"] + assert child_manifest["configuration"] == manifest["configuration"] + assert not (child / "report.md").exists() + assert (child / "artifacts/raw-llm-responses.json").exists() + for finding in json.loads((child / "normalized-llm-findings.json").read_text()): + assert finding["page_url"] == child_manifest["source_url"] + assert finding["run_id"] == child.name + assert finding["finding_id"] not in identities + identities.add(finding["finding_id"]) + assert "Full evaluation complete" in capsys.readouterr().out + + +def test_allowance_exhausted_between_pages_keeps_later_pages_unexecuted( + full_workspace, monkeypatch, +): + """Finishing one page consumes the single allowance rather than resetting it.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", + {"completion_tokens": 1_000_000}, + {"completion_tokens": 1_000_000}) + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "1", + "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + assert len(requests) == 2 + assert not manifest["complete"] and "exhausted" in manifest["incomplete_reason"] + assert manifest["estimated_cost_usd"] == "1" + assert manifest["usage"]["output_tokens"] == 2_000_000 + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "not_executed", "not_executed", + ] + assert [child["complete"] for _, child in children(run, manifest)] == [True, False, False] + assert not (run / manifest["pages"][1]["run_dir"] / "normalized-llm-findings.json").exists() + + +def test_partial_page_consumes_only_remaining_allowance(full_workspace, monkeypatch): + """A second page cannot repeat the first page's full spending allowance.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", + *[{"completion_tokens": 500_000}] * 6) + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "0.75", + "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert len(requests) == 3 + assert manifest["estimated_cost_usd"] == "0.75" + assert manifest["usage"]["output_tokens"] == 1_500_000 + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "incomplete", "not_executed", + ] + assert saved[1][1]["max_cost_usd"] == "0.25" + assert saved[1][1]["estimated_cost_usd"] == "0.25" + assert saved[1][1]["incomplete_reason"] == "cost guardrail exhausted" + assert len(json.loads((saved[1][0] / "artifacts/raw-llm-responses.json").read_text())) == 1 + + +def test_failed_stream_usage_is_retained_and_later_pages_stop(full_workspace, monkeypatch): + """A request failure retains reported usage and partial evidence alongside prior pages.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + requests = recording_provider_usage( + monkeypatch, "claude-haiku-5-5", + {"output_tokens": 1_000}, {"output_tokens": 1_000}, + {"input_tokens": 100, "output_tokens": 2_000}, stream_failure_at=2, + ) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert len(requests) == 3 + assert manifest["estimated_cost_usd"] == "0.00201" + assert manifest["usage"]["input_tokens"] == 100 + assert manifest["usage"]["output_tokens"] == 4_000 + assert "synthetic stream failure" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "incomplete", "not_executed", + ] + assert saved[0][1]["complete"] + assert (saved[0][0] / "normalized-llm-findings.json").exists() + assert (saved[1][0] / "normalized-llm-findings.json").exists() + raw = json.loads((saved[1][0] / "artifacts/raw-llm-responses.json").read_text()) + assert not raw[0]["result"]["success"] + assert raw[0]["result"]["usage"]["output_tokens"] == 2_000 + + +@pytest.mark.parametrize("last_request", [1, 5]) +def test_overshoot_completes_only_when_no_more_work_remains( + full_workspace, monkeypatch, last_request, +): + """The underway request may overshoot; completion depends on remaining work.""" + usages = [{"completion_tokens": 0}] * last_request + [{"completion_tokens": 1_000_000}] + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", *usages) + expected = last_request == 5 + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "0.1", + "--live", "--approve-live") == (0 if expected else 1) + run, manifest = only_full(full_workspace) + assert len(requests) == last_request + 1 + assert manifest["complete"] is expected + assert manifest["estimated_cost_usd"] == "0.5" + assert children(run, manifest)[0][1]["complete"] + if expected: + assert all(page["complete"] for page in manifest["pages"]) + else: + assert "exhausted" in manifest["incomplete_reason"] + + +def test_fresh_full_attempt_repeats_requests_with_new_child_identities( + full_workspace, monkeypatch, +): + """Saved HTML is reused, while every new execution starts with fresh requests.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna") + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + first, first_manifest = only_full(full_workspace) + first_count = len(requests) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + runs = list((full_workspace / ".model-evaluation/full-runs").iterdir()) + assert len(runs) == 2 and len(requests) == first_count * 2 + second = next(run for run in runs if run != first) + second_manifest = json.loads((second / "full.json").read_text()) + assert {page["run_id"] for page in first_manifest["pages"]}.isdisjoint( + page["run_id"] for page in second_manifest["pages"] + ) + + +def test_provider_initialization_failure_records_stopping_reason(full_workspace, monkeypatch): + """Failed provider setup retains prepared evidence and identifies unfinished pages.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + def fail(**options): + raise ValueError("synthetic provider initialization failure") + + monkeypatch.setattr("anthropic.Anthropic", fail) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + assert manifest["estimated_cost_usd"] == "0" + assert "synthetic provider initialization failure" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "incomplete", "not_executed", "not_executed", + ] + assert "synthetic provider initialization failure" in children(run, manifest)[0][1]["incomplete_reason"] + + +def test_noninteractive_approval_requires_live_before_preparing(full_workspace, capsys): + """An approval flag alone cannot create or authorize a Full evaluation.""" + with pytest.raises(SystemExit) as error: + preview(full_workspace, "--approve-live") + assert error.value.code == 2 + assert "--approve-live requires --live" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize("model,key", [ + ("gpt-6-luna", "OPENAI_API_KEY"), ("claude-haiku-5-5", "ANTHROPIC_API_KEY"), +]) +def test_live_requires_the_selected_providers_credentials( + full_workspace, monkeypatch, capsys, model, key, +): + """The other provider's credentials cannot authorize this fixed preset.""" + monkeypatch.setenv("ANTHROPIC_API_KEY" if key == "OPENAI_API_KEY" else "OPENAI_API_KEY", + "synthetic-other-provider") + with pytest.raises(SystemExit) as error: + preview(full_workspace, "--model", model, "--live", "--approve-live") + assert error.value.code == 2 + assert f"requires {key}" in capsys.readouterr().err + run, manifest = only_full(full_workspace) + assert not manifest["complete"] + assert all((child / "review.csv").exists() for child, _ in children(run, manifest)) + + +def test_parent_sums_all_reported_usage_categories(full_workspace, monkeypatch): + """Cache usage remains a separate provider-reported category in parent totals.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + recording_provider_usage(monkeypatch, "claude-haiku-5-5", { + "input_tokens": 100, "output_tokens": 1_000, + "cache_read_input_tokens": 1_000, "cache_creation_input_tokens": 1_000, + }) + assert preview(full_workspace, "--live", "--approve-live") == 0 + _, manifest = only_full(full_workspace) + assert manifest["usage"] == { + "input_tokens": 100, "output_tokens": 1_000, + "cached_input_tokens": 1_000, "cache_creation_input_tokens": 1_000, + } + assert manifest["estimated_cost_usd"] == "0.000645" + + +def test_unreadable_successful_responses_preserve_execution_completion( + full_workspace, monkeypatch, +): + """Unreadable responses yield no findings while retaining current completion semantics.""" + from types import SimpleNamespace + + monkeypatch.setenv("OPENAI_API_KEY", "synthetic") + + def create(**options): + return SimpleNamespace( + id="synthetic-unreadable", model="gpt-6-luna", + choices=[SimpleNamespace(message=SimpleNamespace(content="unreadable"), + finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=0, completion_tokens=0, total_tokens=0, + prompt_tokens_details=None, completion_tokens_details=None), + ) + + monkeypatch.setattr("openai.OpenAI", lambda **_: SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)))) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + assert manifest["complete"] + for child, saved in children(run, manifest): + assert saved["complete"] and len(saved["format_failures"]) == len(saved["prompts"]) + assert json.loads((child / "normalized-llm-findings.json").read_text()) == [] diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index b76567f..827ce34 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -1,4 +1,4 @@ -"""Run one private fixed-model evaluation, then score a human-edited CSV.""" +"""Prepare and execute fixed-model evaluations, then score human decisions.""" from __future__ import annotations @@ -16,7 +16,7 @@ from dotenv import load_dotenv from vision_aid.evaluation.full import ( - capture_snapshots, load_full_inputs, prepare_full_evaluation, + capture_snapshots, execute_full_evaluation, load_full_inputs, prepare_full_evaluation, ) from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( @@ -80,7 +80,7 @@ def build_parser() -> argparse.ArgumentParser: help="explicitly approve billable execution without a prompt", ) run_all = commands.add_parser( - "run-all", help="preview every listed page with one shared cost allowance", + "run-all", help="preview or execute every listed page with one shared allowance", description=run.description, ) run_all.add_argument( @@ -96,7 +96,15 @@ def build_parser() -> argparse.ArgumentParser: ) run_all.add_argument( "--max-cost-usd", required=True, - help="positive total allowance shared across pages; checked between requests", + help=( + "positive total allowance shared across pages; checked between requests; " + "a request underway can exceed it" + ), + ) + run_all.add_argument("--live", action="store_true", help="enable billable execution") + run_all.add_argument( + "--approve-live", action="store_true", + help="explicitly approve all displayed pages without a prompt; requires --live", ) report = commands.add_parser( "report", help="score completed run and edited review.csv" @@ -153,6 +161,16 @@ def _run(args: argparse.Namespace) -> int: if not args.live: print("Preview complete; no paid requests made.") return 0 + api_key = _approved_api_key(args, configuration) + if api_key is None: + return 0 + complete = execute(run, manifest, api_key, request_config) + print(f"Run {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 + + +def _approved_api_key(args: argparse.Namespace, configuration: dict[str, Any]) -> str | None: + """Select the provider's credentials and ask once for the displayed scope.""" load_dotenv(PROJECT_ROOT / ".env") key_name = ( "OPENAI_API_KEY" if configuration["provider"] == "openai" else "ANTHROPIC_API_KEY" @@ -165,10 +183,8 @@ def _run(args: argparse.Namespace) -> int: or input("Type 'yes' to approve paid execution: ") != "yes" ): print("Live execution declined; no paid requests made.") - return 0 - complete = execute(run, manifest, api_key, request_config) - print(f"Run {'complete' if complete else 'incomplete'}: {run}") - return 0 if complete else 1 + return None + return api_key def _display_model( @@ -219,9 +235,11 @@ def _display_model( def _run_all(args: argparse.Namespace) -> int: - """Preview all listed pages using saved snapshots and no provider access.""" + """Prepare all listed pages and approve the entire scope once.""" request_config = MODEL_PRESETS[args.model] limit = positive_cost(args.max_cost_usd) + if args.approve_live and not args.live: + raise ValueError("--approve-live requires --live") audits, snapshots = load_full_inputs(args.human_audit, args.snapshots) run = _new_run_directory("full-runs") print(f"Full evaluation: {run}") @@ -238,8 +256,15 @@ def _run_all(args: argparse.Namespace) -> int: _display_model( request_config, manifest["configuration"], schedule, prompt_count, limit ) - print("Preview complete; no paid requests made.") - return 0 + if not args.live: + print("Preview complete; no paid requests made.") + return 0 + api_key = _approved_api_key(args, manifest["configuration"]) + if api_key is None: + return 0 + complete = execute_full_evaluation(run, manifest, api_key, request_config) + print(f"Full evaluation {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 def _report(args: argparse.Namespace) -> int: diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py index 7e21042..ad34316 100644 --- a/vision_aid/evaluation/full.py +++ b/vision_aid/evaluation/full.py @@ -13,7 +13,7 @@ from vision_aid.evaluation.human_audit import import_all_human_findings, list_audit_urls from vision_aid.evaluation.llm_audit import ( - LLMRequestConfig, USAGE_KEYS, evaluation_configuration, write_json, + LLMRequestConfig, USAGE_KEYS, evaluation_configuration, execute, write_json, ) from vision_aid.evaluation.page import prepare_page from vision_aid.evaluation.schemas import HumanAudit @@ -150,3 +150,51 @@ def prepare_full_evaluation( manifest["incomplete_reason"] = "preview" write_json(run / "full.json", manifest) return manifest + + +def execute_full_evaluation( + run: Path, manifest: dict[str, Any], api_key: str, + request_config: LLMRequestConfig, +) -> bool: + """Execute prepared pages sequentially with one remaining cost allowance.""" + total_cost = Decimal(0) + usage = {key: 0 for key in USAGE_KEYS} + limit = Decimal(manifest["max_cost_usd"]) + manifest["incomplete_reason"] = None + for number, page in enumerate(manifest["pages"]): + remaining = limit - total_cost + if remaining <= 0: + manifest["incomplete_reason"] = "cost guardrail exhausted before next page" + for unfinished in manifest["pages"][number:]: + unfinished["status"] = "not_executed" + write_json(run / "full.json", manifest) + return False + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + child_manifest["max_cost_usd"] = str(remaining) + write_json(child / "run.json", child_manifest) + try: + complete = execute(child, child_manifest, api_key, request_config) + except Exception as error: # noqa: BLE001 + # Provider setup can fail before the executor saves a request outcome. + child_manifest.update(complete=False, incomplete_reason=str(error)) + write_json(child / "run.json", child_manifest) + complete = False + total_cost += Decimal(child_manifest["estimated_cost_usd"]) + for key in USAGE_KEYS: + usage[key] += child_manifest["usage"][key] + page.update(status="complete" if complete else "incomplete", complete=complete) + manifest.update(estimated_cost_usd=str(total_cost), usage=usage) + if not complete: + manifest["incomplete_reason"] = ( + f"page execution failed for {page['source_url']}: " + f"{child_manifest['incomplete_reason']}" + ) + for unfinished in manifest["pages"][number + 1:]: + unfinished["status"] = "not_executed" + write_json(run / "full.json", manifest) + if not complete: + return False + manifest.update(complete=True, incomplete_reason=None) + write_json(run / "full.json", manifest) + return True From 49fdc4ba03acba60588532f83480c6248b9e4005 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 17:00:07 -0400 Subject: [PATCH 079/126] test(evaluation): isolate missing provider credentials from dotenv --- tests/test_full_execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py index d0bf469..6fece81 100644 --- a/tests/test_full_execution.py +++ b/tests/test_full_execution.py @@ -218,6 +218,7 @@ def test_live_requires_the_selected_providers_credentials( full_workspace, monkeypatch, capsys, model, key, ): """The other provider's credentials cannot authorize this fixed preset.""" + monkeypatch.setenv(key, "") monkeypatch.setenv("ANTHROPIC_API_KEY" if key == "OPENAI_API_KEY" else "OPENAI_API_KEY", "synthetic-other-provider") with pytest.raises(SystemExit) as error: From 3c3000ee0d786d8d404a4b4625f5402381b85db6 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 17:08:28 -0400 Subject: [PATCH 080/126] feat(evaluation): report combined detection for every listed page --- README.md | 53 +++++ tests/test_full_report.py | 353 ++++++++++++++++++++++++++++++++ vision_aid/evaluation/cli.py | 20 +- vision_aid/evaluation/full.py | 145 ++++++++++++- vision_aid/evaluation/review.py | 77 +++++++ 5 files changed, 646 insertions(+), 2 deletions(-) create mode 100644 tests/test_full_report.py diff --git a/README.md b/README.md index 0a4c397..4f01b04 100644 --- a/README.md +++ b/README.md @@ -603,6 +603,59 @@ and child directories and repeats every model request, including pages that completed in a previous attempt. It reuses captured HTML without downloading again; it does not resume or reuse previous paid outputs. +### Review and report a Full evaluation + +After every page completes execution, edit each child's `review.csv` under +`.model-evaluation/full-runs//pages/`. Use the same Eligibility +classifications, finding ID columns, and full Match decisions as the one-page +workflow below. Establish eligibility from the Human finding and saved HTML +independently of what the model caught. Each page includes all Global findings, +so their evidence and eligibility can be reviewed separately on every page. + +Generate the individual page reports and the combined report: + +```bash +uv run visionaid-evaluate report-all \ + --run-dir .model-evaluation/full-runs/ \ + --reviewer "Name and how the page decisions were reviewed" +``` + +`report-all` validates every listed execution, child evidence identity, and +page review before writing final combined rates to the parent's `report.md`. +The report lists every original Human finding once, identifies the reviewer, +links page reports, names supporting pages and finding IDs, and totals saved +estimated cost, provider-reported usage, and Unreadable responses. The supplied +audit has 86 originals across 17 pages; repeated Globals produce 198 review +rows, while the combined Overall denominator stays 86. + +A Global earns combined credit when one page's selected findings fully cover +it. Several findings from one source on that page may jointly cover it. +Partial evidence on different pages never earns combined credit. LLM and +Programmatic catches may come from different pages, but Overall counts that +Global once. Repeated catches on several pages also count once. + +Global classifications may be `unavailable_evidence` or `ambiguous` on some +pages. If any page is testable, all testable pages must agree on `llm_eligible` +or `programmatic`; that original finding enters the corresponding denominator +once, including when nothing caught it. If every page is not testable, its +recorded reasons remain in the report, it leaves the LLM and Programmatic +denominators, and it stays in Overall. Combined rates use pooled finding +counts, rather than averages of page percentages. A catch by the other source +counts in Overall but earns no credit in the expected source's rate. + +Missing or incomplete execution, invalid reviews, missing evidence, or +conflicting testable Global classifications withhold final combined rates. +Correct review decisions and run `report-all` again. Regenerating a report +removes the previous parent report before validation, so invalid revised +reviews cannot leave stale combined rates. Available individual reports and +evidence remain. To report one completed child after a later page failure, use: + +```bash +uv run visionaid-evaluate report \ + --run-dir .model-evaluation/full-runs//pages/ \ + --reviewer "Name and how this page was reviewed" +``` + ### How the evaluator reads LLM responses The evaluator reads each LLM response using va-dat's own parsing and reporting diff --git a/tests/test_full_report.py b/tests/test_full_report.py new file mode 100644 index 0000000..e03b97f --- /dev/null +++ b/tests/test_full_report.py @@ -0,0 +1,353 @@ +"""Report pooled Full evaluation results through the public CLI.""" + +import csv +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import openpyxl +import pytest + +from test_evaluation import recording_provider_usage +from test_full_execution import children +from test_full_preview import URLS, full_workspace, only_full, preview +from vision_aid.evaluation.cli import main + + +def save_json(path, value): + """Save a deliberate evidence mutation or synthetic input.""" + path.write_text(json.dumps(value), encoding="utf-8") + + +def review_rows(child): + """Read the operator-editable decisions for one page.""" + with (child / "review.csv").open(newline="") as stream: + return list(csv.DictReader(stream)) + + +def save_review(child, rows): + """Save reviewer decisions while keeping all original columns.""" + path = child / "review.csv" + with path.open(newline="") as stream: + fields = csv.DictReader(stream).fieldnames + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + + +def report_all(run, reviewer="Combined reviewer"): + """Invoke deterministic combined reporting with reviewer attribution.""" + return main(["report-all", "--run-dir", str(run), "--reviewer", reviewer]) + + +def refused_report(run, capsys, reviewer="Combined reviewer"): + """Assert invalid evidence cannot publish a final combined report.""" + with pytest.raises(SystemExit) as error: + report_all(run, reviewer) + assert error.value.code == 2 + assert not (run / "report.md").exists() + return capsys.readouterr().err + + +@pytest.fixture +def completed_review(full_workspace, monkeypatch): + """Execute uneven page memberships, then record hand-calculated decisions.""" + path = full_workspace / "human-audit.xlsx" + book = openpyxl.load_workbook(path) + sheet = book["Defect report"] + for number, title in enumerate( + ("Eligible uncaught", "Partial across pages", "Joint catch"), start=6, + ): + sheet.append([number, "Element", "Chrome", "Global", title, + f"Open {URLS[0]}", "Original", "Expected", "", "1.1.1"]) + book.save(path) + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + save_json(mapping_path, mapping) + recording_provider_usage(monkeypatch, "gpt-6-luna", { + "prompt_tokens": 1000, "completion_tokens": 100, + }) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + for number, (child, _) in enumerate(children(run, manifest)): + llm = json.loads((child / "normalized-llm-findings.json").read_text()) + programmatic = json.loads((child / "normalized-programmatic-findings.json").read_text()) + rows = review_rows(child) + for row in rows: + row.update(classification="llm_eligible", classification_reason="HTML is sufficient") + title = row["Issue Title"] + if title == "First local": + row.update(llm_finding_id=llm[0]["finding_id"], match_reason="Complete first issue") + elif title == "Second local": + row.update(classification="programmatic", llm_finding_id=llm[1]["finding_id"], + match_reason="Cross-catch by LLM") + elif title == "Contact local": + row.update(programmatic_finding_id=programmatic[0]["finding_id"], + match_reason="Cross-catch by Programmatic checks") + elif title == "Shared defect": + if number == 0: + row.update(programmatic_finding_id=programmatic[0]["finding_id"], + match_reason="Complete Global on home") + else: + identities = llm if number == 1 else llm[:1] + row.update(llm_finding_id=";".join(item["finding_id"] for item in identities), + match_reason="Complete Global on another page") + elif title == "Shared second defect": + row.update(classification="ambiguous" if number == 1 else "unavailable_evidence", + classification_reason=f"Not testable on page {number + 1}") + elif title == "Eligible uncaught": + row.update(classification="programmatic" if number == 1 else "unavailable_evidence", + classification_reason=f"Evidence varies on page {number + 1}") + elif title == "Partial across pages": + row.update(review_notes=f"Partial evidence only on page {number + 1}") + elif title == "Joint catch": + row.update(classification="programmatic") + if number == 1: + row.update(programmatic_finding_id=";".join( + item["finding_id"] for item in programmatic[1:3]), + match_reason="These two findings jointly cover the entire Global") + save_review(child, rows) + return run, manifest + + +def test_combined_report_pools_original_findings_and_any_page_globals(completed_review): + """Eight originals score 2/4 LLM, 1/3 Programmatic and 5/8 Overall.""" + run, manifest = completed_review + assert report_all(run) == 0 + text = (run / "report.md").read_text() + assert "LLM detection rate: Of the 4" in text and "caught 2 (50.0%)" in text + assert "Programmatic detection rate: Of the 3" in text and "caught 1 (33.3%)" in text + assert "Overall detection rate: Of all 8 Human findings" in text and "caught 5 (62.5%)" in text + assert "Review decisions by: Combined reviewer" in text + assert "Estimated run cost: $0.00" in text + assert "1000 input tokens" in text and "100 output tokens" in text + assert "published prices captured on 2026-10-09" in text + assert "Unreadable responses: 0" in text + originals = json.loads((run / "human-findings.json").read_text()) + for finding in originals["human_findings"]: + assert text.count(finding["human_finding_id"]) == 1 + assert "Not testable on page 1" in text and "Not testable on page 2" in text + assert "Partial evidence only on page 1" in text + for page in manifest["pages"]: + child = run / page["run_dir"] + assert (child / "report.md").is_file() + assert f"]({page['run_dir']}/report.md)" in text + assert page["source_url"] in text + for row in review_rows(child): + for kind in ("llm", "programmatic"): + for identity in row[f"{kind}_finding_id"].split(";"): + if identity: + assert identity in text + assert "complete detection on any single page" in text + assert "partial matches across pages" in text + + +@pytest.mark.parametrize("problem", [ + "parent_incomplete", "parent_id", "missing_page", "duplicate_page", "reordered_pages", + "child_parent_id", "expected_ids", "child_configuration", "human_evidence", + "missing_snapshot", "snapshot_bytes", "pricing_bytes", "missing_raw_llm", + "parent_cost", "parent_usage", +]) +def test_report_requires_complete_membership_and_owned_evidence(completed_review, problem, capsys): + """Completed flags cannot hide omitted pages or altered child evidence.""" + run, manifest = completed_review + first = run / manifest["pages"][0]["run_dir"] + if problem == "parent_incomplete": + manifest["complete"] = False + elif problem == "parent_id": + manifest["run_id"] = "foreign-full-evaluation" + elif problem == "missing_page": + manifest["pages"].pop() + elif problem == "duplicate_page": + manifest["pages"].append(manifest["pages"][0]) + elif problem == "reordered_pages": + manifest["pages"].reverse() + elif problem == "child_parent_id": + manifest["pages"][0]["run_id"] = "foreign-child" + elif problem == "expected_ids": + manifest["pages"][0]["human_finding_ids"].pop() + elif problem == "child_configuration": + saved = json.loads((first / "run.json").read_text()) + saved["configuration"]["model"] = "claude-haiku-5-5" + save_json(first / "run.json", saved) + elif problem == "human_evidence": + path = first / "artifacts/human-findings.json" + saved = json.loads(path.read_text()) + saved["human_findings"][0]["problem"] = "Replaced Human finding evidence" + save_json(path, saved) + elif problem == "missing_snapshot": + (first / "snapshot.html").unlink() + elif problem == "snapshot_bytes": + (first / "snapshot.html").write_text("another page") + elif problem == "pricing_bytes": + path = first / "artifacts/pricing.json" + path.write_text(path.read_text() + " ") + elif problem == "missing_raw_llm": + (first / "artifacts/raw-llm-responses.json").unlink() + elif problem == "parent_cost": + manifest["estimated_cost_usd"] = "7" + elif problem == "parent_usage": + manifest["usage"]["output_tokens"] = 99999 + save_json(run / "full.json", manifest) + refused_report(run, capsys) + + +def test_empty_page_without_globals_still_has_a_report(full_workspace, monkeypatch): + """A listed page with no Human findings can complete its empty review.""" + path = full_workspace / "human-audit.xlsx" + book = openpyxl.load_workbook(path) + book["Defect report"].delete_rows(5, 2) + book.save(path) + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + save_json(mapping_path, mapping) + recording_provider_usage(monkeypatch, "gpt-6-luna") + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + for child, _ in children(run, manifest): + rows = review_rows(child) + for row in rows: + row.update(classification="unavailable_evidence", classification_reason="Needs a browser") + save_review(child, rows) + assert report_all(run) == 0 + empty = run / manifest["pages"][2]["run_dir"] + text = (empty / "report.md").read_text() + assert "Overall detection rate: No Human findings were imported for this page" in text + assert "Overall detection rate: Of all 3 Human findings" in (run / "report.md").read_text() + + +def test_invalid_revised_review_withholds_stale_combined_rates(completed_review, capsys): + """A report regenerated after an invalid edit must not leave earlier rates.""" + run, manifest = completed_review + assert report_all(run) == 0 + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + rows[0]["classification"] = "" + save_review(first, rows) + assert "invalid classification" in refused_report(run, capsys) + assert (first / "report.md").exists() + + +@pytest.mark.parametrize("title", ["Shared defect", "Eligible uncaught"]) +def test_conflicting_testable_global_classifications_require_correction(completed_review, capsys, title): + """Eligibility conflicts block reporting whether or not the Global was caught.""" + run, manifest = completed_review + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + row = next(row for row in rows if row["Issue Title"] == title) + row["classification"] = "programmatic" if title == "Shared defect" else "llm_eligible" + save_review(first, rows) + assert "conflicting testable classifications" in refused_report(run, capsys) + + +@pytest.mark.parametrize("problem", [ + "unclassified", "source_identity", "wrong_source_id", "foreign_id", + "missing_reason", "reused_id", "missing_child", "missing_review", +]) +def test_every_page_review_uses_the_existing_validation(completed_review, capsys, problem): + """Unfinished or invalid page decisions cannot receive pooled credit.""" + run, manifest = completed_review + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + if problem == "unclassified": + rows[0]["classification"] = "" + elif problem == "source_identity": + rows[0]["source_element"] = "Changed source element" + elif problem == "wrong_source_id": + rows[0]["llm_finding_id"] = json.loads( + (first / "normalized-programmatic-findings.json").read_text())[0]["finding_id"] + elif problem == "foreign_id": + other = run / manifest["pages"][1]["run_dir"] + rows[0]["llm_finding_id"] = json.loads( + (other / "normalized-llm-findings.json").read_text())[0]["finding_id"] + elif problem == "missing_reason": + rows[0]["match_reason"] = "" + elif problem == "reused_id": + rows[1]["llm_finding_id"] = rows[0]["llm_finding_id"] + elif problem == "missing_child": + (first / "run.json").unlink() + elif problem == "missing_review": + (first / "review.csv").unlink() + if problem not in {"missing_child", "missing_review"}: + save_review(first, rows) + refused_report(run, capsys) + + +def test_no_eligible_sources_retain_all_originals_in_overall(completed_review): + """All-not-testable reviews produce no source denominator and zero of eight Overall.""" + run, manifest = completed_review + for child, _ in children(run, manifest): + rows = review_rows(child) + for row in rows: + row.update(classification="unavailable_evidence", classification_reason="No saved evidence", + llm_finding_id="", programmatic_finding_id="", match_reason="") + save_review(child, rows) + assert report_all(run) == 0 + text = (run / "report.md").read_text() + assert "LLM detection rate: No Human findings were expected" in text + assert "Programmatic detection rate: No Human findings were expected" in text + assert "Overall detection rate: Of all 8 Human findings" in text + assert "caught 0 (0.0%)" in text + + +def test_completed_child_report_survives_later_parent_failure(full_workspace, monkeypatch, capsys): + """An incomplete parent withholds combined rates while its finished page reports.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + recording_provider_usage(monkeypatch, "claude-haiku-5-5", stream_failure_at=2) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + for row in rows: + row.update(classification="llm_eligible", classification_reason="Saved evidence") + save_review(first, rows) + assert main(["report", "--run-dir", str(first), "--reviewer", "Page reviewer"]) == 0 + saved = (first / "report.md").read_bytes() + assert "Incomplete Full evaluations" in refused_report(run, capsys) + assert (first / "report.md").read_bytes() == saved + + +def test_reviewer_description_is_required(completed_review, capsys): + """Scoring cannot discard the identity of the reviewer making the decisions.""" + run, _ = completed_review + assert "reviewer description" in refused_report(run, capsys, reviewer=" ") + + +def test_combined_report_summarizes_unreadable_responses(full_workspace, monkeypatch): + """Successful Unreadable responses remain complete and are counted across pages.""" + monkeypatch.setenv("OPENAI_API_KEY", "synthetic") + + def create(**options): + """Return an unreadable provider reply without leaving the process.""" + return SimpleNamespace( + id="unreadable", model="gpt-6-luna", + choices=[SimpleNamespace(message=SimpleNamespace(content="not JSON"), finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=0, completion_tokens=0, total_tokens=0, + prompt_tokens_details=None, completion_tokens_details=None), + ) + + monkeypatch.setattr("openai.OpenAI", lambda **_: SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)))) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + for child, _ in children(run, manifest): + rows = review_rows(child) + for row in rows: + row.update(classification="llm_eligible", classification_reason="HTML evidence") + save_review(child, rows) + assert report_all(run) == 0 + assert "Unreadable responses: 6 of 6 LLM checks" in (run / "report.md").read_text() + + +def test_report_all_help_explains_review_completion(capsys): + """The combined reporting command explains when it can publish scores.""" + with pytest.raises(SystemExit) as error: + main(["report-all", "--help"]) + assert error.value.code == 0 + text = " ".join(capsys.readouterr().out.split()) + assert "every listed execution" in text and "all page reviews" in text + assert "Global" in text and "once" in text diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 827ce34..95d42d4 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -16,7 +16,8 @@ from dotenv import load_dotenv from vision_aid.evaluation.full import ( - capture_snapshots, execute_full_evaluation, load_full_inputs, prepare_full_evaluation, + capture_snapshots, execute_full_evaluation, load_full_inputs, + prepare_full_evaluation, report_full_evaluation, ) from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( @@ -111,6 +112,18 @@ def build_parser() -> argparse.ArgumentParser: ) report.add_argument("--run-dir", type=Path, required=True) report.add_argument("--reviewer", required=True, help="who made the review decisions") + report_all = commands.add_parser( + "report-all", help="validate every page review and score unique Human findings", + description=( + "Final combined reporting requires every listed execution to complete " + "and all page reviews to validate. Each original Human finding counts " + "once. A Global finding earns source credit through complete detection " + "on any single page; testable Global classifications must agree. " + "Individual completed page reports remain available after parent failure." + ), + ) + report_all.add_argument("--run-dir", type=Path, required=True, help="Full evaluation directory") + report_all.add_argument("--reviewer", required=True, help="who made all page review decisions") return parser @@ -286,6 +299,11 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 if args.command == "run-all": return _run_all(args) + if args.command == "report-all": + run = args.run_dir.resolve() + report_full_evaluation(run, args.reviewer) + print(f"Combined Markdown report: {run / 'report.md'}") + return 0 return _run(args) if args.command == "run" else _report(args) except (OSError, ValueError, KeyError) as error: parser.error(str(error)) diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py index ad34316..8aef3ce 100644 --- a/vision_aid/evaluation/full.py +++ b/vision_aid/evaluation/full.py @@ -15,7 +15,8 @@ from vision_aid.evaluation.llm_audit import ( LLMRequestConfig, USAGE_KEYS, evaluation_configuration, execute, write_json, ) -from vision_aid.evaluation.page import prepare_page +from vision_aid.evaluation.page import PageReview, prepare_page, validated_page_review +from vision_aid.evaluation.review import write_combined_report, write_report from vision_aid.evaluation.schemas import HumanAudit @@ -126,6 +127,7 @@ def prepare_full_evaluation( "human_audit_sha256": first_audit.human_audit_sha256, "human_findings": sorted(originals.values(), key=lambda item: item["source_row"]), "page_assignments": assignments, + "source_urls": list(audits), }) write_json(run / "full.json", manifest) for page in pages: @@ -198,3 +200,144 @@ def execute_full_evaluation( manifest.update(complete=True, incomplete_reason=None) write_json(run / "full.json", manifest) return True + + +def _pooled_rows( + originals: list[dict[str, Any]], + pages: list[dict[str, Any]], + reviews: list[PageReview], +) -> list[dict[str, Any]]: + """Keep each original once, using complete source matches on any one page.""" + grouped: dict[str, list[dict[str, Any]]] = {} + for page, review in zip(pages, reviews, strict=True): + for row in review.rows: + grouped.setdefault(row["human_finding"]["human_finding_id"], []).append({ + "source_url": page["source_url"], + "decision": row["decision"], + "matches": row["matches"], + }) + pooled = [] + for finding in originals: + page_rows = grouped[finding["human_finding_id"]] + classifications = { + row["decision"]["classification"].strip() for row in page_rows + if row["decision"]["classification"].strip() in {"llm_eligible", "programmatic"} + } + if len(classifications) > 1: + raise ValueError( + f"Global Human finding {finding['human_finding_id']} has conflicting " + "testable classifications; all testable pages must agree" + ) + classification = ( + next(iter(classifications)) if classifications + else page_rows[0]["decision"]["classification"].strip() + ) + pooled.append({ + "human_finding": finding, + "decision": {"classification": classification}, + "matches": { + kind: tuple(identity for row in page_rows for identity in row["matches"][kind]) + for kind in ("llm", "programmatic") + }, + "page_reviews": page_rows, + }) + return pooled + + +def _validated_full_reviews( + run: Path, manifest: dict[str, Any], original: dict[str, Any], +) -> list[PageReview]: + """Check ordered membership, original evidence and ownership for every child.""" + if manifest["run_id"] != run.name: + raise ValueError("Full evaluation evidence does not belong to this directory") + if not manifest["complete"]: + raise ValueError("Incomplete Full evaluations cannot produce a final report") + urls = original["source_urls"] + if not urls or len(urls) != len(set(urls)) or [ + page["source_url"] for page in manifest["pages"] + ] != urls: + raise ValueError("Full evaluation membership differs from the original ordered targets") + if original["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the Full evaluation") + findings = original["human_findings"] + identities = [item["human_finding_id"] for item in findings] + if not identities or len(identities) != len(set(identities)): + raise ValueError("Full evaluation must preserve each original Human finding once") + assignments = original["page_assignments"] + locals_ = {item["human_finding_id"] for item in findings if item["page_scope"] != "global"} + if set(assignments) != locals_ or any(url not in urls for url in assignments.values()): + raise ValueError("Full evaluation has invalid original page assignments") + reviews = [] + child_ids: set[str] = set() + tool_ids: set[str] = set() + for page in manifest["pages"]: + child = (run / page["run_dir"]).resolve() + if ( + page["run_id"] in child_ids + or page["run_dir"] != f"pages/{page['run_id']}" + or child.parent != run / "pages" + or child.name != page["run_id"] + ): + raise ValueError("Child Evaluation run identity or directory ownership differs") + child_ids.add(page["run_id"]) + if not page["complete"] or page["status"] != "complete": + raise ValueError("Every listed page must complete execution before combined reporting") + expected = [ + item for item in findings if item["page_scope"] == "global" + or assignments[item["human_finding_id"]] == page["source_url"] + ] + if page["human_finding_ids"] != [item["human_finding_id"] for item in expected]: + raise ValueError("Child Human finding membership differs from its original assignments") + evidence = json.loads((child / "artifacts/human-findings.json").read_text()) + if evidence["human_findings"] != expected or evidence["homepage_url"] != page["source_url"]: + raise ValueError("Child Human finding evidence differs from the original page selection") + review = validated_page_review(child) + saved = review.manifest + for field, expected_value in ( + ("run_id", page["run_id"]), ("source_url", page["source_url"]), + ("human_audit_sha256", manifest["human_audit_sha256"]), + ("configuration", manifest["configuration"]), + ("snapshot_sha256", page["snapshot_sha256"]), + ): + if saved[field] != expected_value: + raise ValueError(f"Child {field} differs from the Full evaluation") + if hashlib.sha256((child / "snapshot.html").read_bytes()).hexdigest() != saved["snapshot_sha256"]: + raise ValueError("Child Benchmark snapshot checksum differs from the saved evidence") + if review.schedule.identity != saved["pricing_sha256"] or review.schedule.version != saved["pricing_version"]: + raise ValueError("Child pricing evidence differs from its recorded schedule") + for filename in ("raw-llm-responses.json", "raw-programmatic-findings.json"): + json.loads((child / "artifacts" / filename).read_text()) + for prompt in saved["prompts"]: + json.loads((child / "artifacts/prompts" / f"{prompt['name']}.json").read_text()) + if any(item["model"] != manifest["configuration"]["model"] for item in review.llm): + raise ValueError("LLM findings do not belong to the selected model") + ids = [item["finding_id"] for item in review.llm + review.programmatic] + if len(ids) != len(set(ids)) or tool_ids.intersection(ids): + raise ValueError("Tool finding identities must be unique across all page runs") + tool_ids.update(ids) + reviews.append(review) + cost = sum((Decimal(review.manifest["estimated_cost_usd"]) for review in reviews), Decimal(0)) + usage = {key: sum(review.manifest["usage"][key] for review in reviews) for key in USAGE_KEYS} + if cost != Decimal(manifest["estimated_cost_usd"]) or usage != manifest["usage"]: + raise ValueError("Full evaluation totals differ from authoritative child cost and usage") + return reviews + + +def report_full_evaluation(run: Path, reviewer: str) -> None: + """Validate every page review before writing page and combined reports.""" + if not reviewer.strip(): + raise ValueError("A reviewer description is required") + run = run.resolve() + # Regeneration cannot leave a previous final report beside invalid decisions. + (run / "report.md").unlink(missing_ok=True) + manifest = json.loads((run / "full.json").read_text()) + original = json.loads((run / "human-findings.json").read_text()) + reviews = _validated_full_reviews(run, manifest, original) + rows = _pooled_rows(original["human_findings"], manifest["pages"], reviews) + for page, review in zip(manifest["pages"], reviews, strict=True): + write_report( + run / page["run_dir"] / "report.md", review.manifest, + review.rows, review.llm, review.programmatic, reviewer, + review.schedule.price_description, + ) + write_combined_report(run / "report.md", manifest, rows, reviews, reviewer) diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index e1569b7..5545f87 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -123,6 +123,8 @@ def reviewed_rows( def _rate(part: str, caught: int, total: int) -> str: """Explain each denominator, including when there is nothing to score.""" if part == "Overall": + if total == 0: + return "Overall detection rate: No Human findings were imported for this page." return f"Overall detection rate: Of all {total} Human findings, either part of the tool caught {caught} ({caught / total:.1%})." label = part.removesuffix(" checks") if total == 0: @@ -284,3 +286,78 @@ def write_report( lines.append(f"- {label}: {_md(decision[field])}") lines.append("") path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def write_combined_report( + path: Path, + manifest: dict[str, Any], + rows: list[dict[str, Any]], + page_reviews: list[Any], + reviewer: str, +) -> None: + """Describe pooled original findings with every supporting page decision.""" + counts = detection_counts(rows) + usage = manifest["usage"] + tokens = [f"{usage['input_tokens']} input tokens", f"{usage['output_tokens']} output tokens"] + for key, label in ( + ("cached_input_tokens", "cached input"), + ("cache_creation_input_tokens", "cache-creation input"), + ): + if usage[key]: + tokens.append(f"{usage[key]} {label} tokens") + prices = ", ".join(dict.fromkeys(review.schedule.price_description for review in page_reviews)) + failures = sum(len(review.manifest["format_failures"]) for review in page_reviews) + prompts = sum(len(review.manifest["prompts"]) for review in page_reviews) + lines = [ + "# Full accessibility evaluation", "", + f"Model: {_md(manifest['configuration']['model'])}. Run ID: {_md(manifest['run_id'])}.", + f"Human audit: {_md(manifest['human_audit_filename'])}; SHA-256: {_md(manifest['human_audit_sha256'])}.", + f"Review decisions by: {_md(reviewer)}", "", + "The tool evaluated only saved HTML for every page listed in the Human audit, with one fixed model. LLM checks use model judgment; Programmatic checks apply fixed rules to HTML.", + "Each original Human finding counts once. Non-Global findings use their assigned page. Global findings earn credit through complete detection on any single page, independently for each detection source. Several catches still count once; partial matches across pages are never combined.", + "Eligibility is established independently of detection. If any page is testable for a Global, the finding enters that expected source's denominator once; all testable pages must agree on the source. Findings not testable on any page remain in Overall but are excluded from LLM and Programmatic rates. Cross-catches count toward Overall, which counts either source's catch once.", + "", "## Summary", "", + ] + for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks"), ("overall", "Overall")): + lines.extend([ + _rate(label, counts[kind]["caught"], counts[kind]["eligible"]).replace( + "the page's HTML", "the evaluated pages' HTML" + ), "", + ]) + lines.extend([ + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from provider-reported usage using {_md(prices)}. Reported totals: {', '.join(tokens)}. The actual bill may differ.", + f"Unreadable responses: {failures} of {prompts} LLM checks. These responses yield no LLM findings and do not change execution completion.", + "", "## Evaluated pages", "", + ]) + for page in manifest["pages"]: + lines.append(f"- [{_md(page['source_url'])}]({page['run_dir']}/report.md)") + lines.extend(["", "## Original Human findings", ""]) + expected = { + "llm_eligible": "LLM", "programmatic": "Programmatic checks", + "unavailable_evidence": "Not testable from the page file", + "ambiguous": "Not testable from the page file", + } + for row in rows: + finding = row["human_finding"] + classification = row["decision"]["classification"] + marks = {kind: "✓" if row["matches"][kind] else "✗" for kind in ("llm", "programmatic")} + lines.extend([ + f"### {_md(finding['problem'])}", "", + f"Human finding ID: {_md(finding['human_finding_id'])}. Source: {_md(finding['source_sheet'])}, row {finding['source_row']}; scope: {_md(finding['page_scope'])}; WCAG: {_md(finding['raw_evidence']['WCAG Sc'])}.", + f"Expected to be caught by: {expected[classification]}. LLM: {marks['llm']}; Programmatic checks: {marks['programmatic']}.", "", + ]) + for page_row in row["page_reviews"]: + decision = page_row["decision"] + source_url = _md(page_row["source_url"]) + lines.append( + f"- Page {source_url}: {expected[decision['classification'].strip()]}" + + (f". {_md(decision['classification_reason'])}" if decision['classification_reason'].strip() else "") + ) + for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks")): + identities = page_row["matches"][kind] + if identities: + lines.append(f" - {label} full match: {', '.join(_md(identity) for identity in identities)}. Why this counts: {_md(decision['match_reason'])}") + if decision["review_notes"].strip(): + lines.append(f" - Review notes: {_md(decision['review_notes'])}") + lines.append("") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") From 51378c5a789a2bbec05c468aa0408a8b641b55ad Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Sat, 10 Oct 2026 21:44:24 -0400 Subject: [PATCH 081/126] refactor(evaluation): address full evaluation review findings --- README.md | 4 ++-- tests/test_evaluation_capture.py | 3 +++ tests/test_full_execution.py | 5 +++++ vision_aid/evaluation/review.py | 32 +++++++++++++++++--------------- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 4f01b04..4271b2f 100644 --- a/README.md +++ b/README.md @@ -538,7 +538,7 @@ Prepare every listed page from the saved Benchmark snapshots: uv run visionaid-evaluate run-all \ --human-audit "Pristine Accessibility Defect Report.xlsx" \ --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ - --model claude-haiku-5-5 \ + --model gpt-6-luna \ --max-cost-usd 5 ``` @@ -573,7 +573,7 @@ selected provider's API key: uv run visionaid-evaluate run-all \ --human-audit "Pristine Accessibility Defect Report.xlsx" \ --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ - --model claude-haiku-5-5 \ + --model gpt-6-luna \ --max-cost-usd 5 \ --live ``` diff --git a/tests/test_evaluation_capture.py b/tests/test_evaluation_capture.py index 7863b7d..2d8fd3f 100644 --- a/tests/test_evaluation_capture.py +++ b/tests/test_evaluation_capture.py @@ -24,6 +24,7 @@ def audit(path: Path, *urls: str | None) -> Path: def capture(human_audit: Path, output: Path) -> int: + """Invoke capture through the public CLI with the supplied audit and output.""" return main(["capture", "--human-audit", str(human_audit), "--output-dir", str(output)]) @@ -37,6 +38,7 @@ def test_capture_saves_only_listed_urls_in_order_and_preserves_redirect_identity downloads = [] def get(url, **options): + """Record downloads and return separate HTML with a redirected URL.""" downloads.append((url, options)) response = requests.Response() response.status_code = 200 @@ -80,6 +82,7 @@ def test_capture_stops_after_http_error_and_keeps_saved_evidence( downloads = [] def get(url, **options): + """Return saved HTML first, then the requested HTTP error.""" downloads.append(url) response = requests.Response() response.status_code = 200 if len(downloads) == 1 else status diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py index 6fece81..fc73629 100644 --- a/tests/test_full_execution.py +++ b/tests/test_full_execution.py @@ -9,6 +9,7 @@ def children(run, manifest): + """Load each page's child directory and manifest in evaluation order.""" return [(run / page["run_dir"], json.loads((run / page["run_dir"] / "run.json").read_text())) for page in manifest["pages"]] @@ -22,6 +23,7 @@ def test_declined_full_approval_makes_no_requests_after_showing_all_pages( approvals = [] def decline(prompt): + """Verify the prepared scope is displayed before declining approval.""" approvals.append(prompt) run, manifest = only_full(full_workspace) assert all((child / "review.csv").exists() for child, _ in children(run, manifest)) @@ -49,6 +51,7 @@ def test_one_approval_executes_ordered_pages_with_one_fixed_preset( approvals = [] def approve(prompt): + """Approve once after every page is prepared and before requests begin.""" assert not requests approvals.append(prompt) run, manifest = only_full(full_workspace) @@ -189,6 +192,7 @@ def test_provider_initialization_failure_records_stopping_reason(full_workspace, monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") def fail(**options): + """Simulate failure while initializing the selected provider.""" raise ValueError("synthetic provider initialization failure") monkeypatch.setattr("anthropic.Anthropic", fail) @@ -255,6 +259,7 @@ def test_unreadable_successful_responses_preserve_execution_completion( monkeypatch.setenv("OPENAI_API_KEY", "synthetic") def create(**options): + """Return a successful provider response that yields no readable findings.""" return SimpleNamespace( id="synthetic-unreadable", model="gpt-6-luna", choices=[SimpleNamespace(message=SimpleNamespace(content="unreadable"), diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 5545f87..48e6f15 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -167,6 +167,21 @@ def _md(value: object) -> str: return text.replace("\n", "
") +def _usage_description(usage: dict[str, int]) -> str: + """Describe reported input, output, and nonzero cache token totals.""" + tokens = [ + f"{usage['input_tokens']} input tokens", + f"{usage['output_tokens']} output tokens", + ] + for key, label in ( + ("cached_input_tokens", "cached input"), + ("cache_creation_input_tokens", "cache-creation input"), + ): + if usage[key]: + tokens.append(f"{usage[key]} {label} tokens") + return ", ".join(tokens) + + def write_report( path: Path, manifest: dict[str, Any], @@ -193,11 +208,6 @@ def write_report( if urlsplit(manifest["source_url"]).path in {"", "/"} else f"This evaluation covers the saved page at {_md(manifest['source_url'])}" ) - usage = manifest["usage"] - tokens = [f"{usage['input_tokens']} input tokens", f"{usage['output_tokens']} output tokens"] - for key, label in (("cached_input_tokens", "cached input"), ("cache_creation_input_tokens", "cache-creation input")): - if usage[key]: - tokens.append(f"{usage[key]} {label} tokens") lines = [ f"# LLM accessibility evaluation: {_md(manifest['source_url'])}", "", @@ -225,7 +235,7 @@ def write_report( lines.extend([ _rate("Overall", counts["overall"]["caught"], counts["overall"]["eligible"]), "", - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated using provider-reported token usage and the {_md(pricing_description)}. Reported totals: {', '.join(tokens)}. The actual bill may differ slightly.", + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated using provider-reported token usage and the {_md(pricing_description)}. Reported totals: {_usage_description(manifest['usage'])}. The actual bill may differ slightly.", "", ]) if manifest["format_failures"]: @@ -297,14 +307,6 @@ def write_combined_report( ) -> None: """Describe pooled original findings with every supporting page decision.""" counts = detection_counts(rows) - usage = manifest["usage"] - tokens = [f"{usage['input_tokens']} input tokens", f"{usage['output_tokens']} output tokens"] - for key, label in ( - ("cached_input_tokens", "cached input"), - ("cache_creation_input_tokens", "cache-creation input"), - ): - if usage[key]: - tokens.append(f"{usage[key]} {label} tokens") prices = ", ".join(dict.fromkeys(review.schedule.price_description for review in page_reviews)) failures = sum(len(review.manifest["format_failures"]) for review in page_reviews) prompts = sum(len(review.manifest["prompts"]) for review in page_reviews) @@ -325,7 +327,7 @@ def write_combined_report( ), "", ]) lines.extend([ - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from provider-reported usage using {_md(prices)}. Reported totals: {', '.join(tokens)}. The actual bill may differ.", + f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from provider-reported usage using {_md(prices)}. Reported totals: {_usage_description(manifest['usage'])}. The actual bill may differ.", f"Unreadable responses: {failures} of {prompts} LLM checks. These responses yield no LLM findings and do not change execution completion.", "", "## Evaluated pages", "", ]) From 3e262ac0c962cf7638e8186a5ca1d5ffa3e1f340 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:32:33 -0400 Subject: [PATCH 082/126] chore(evaluator): add Ruff and fix lint issues --- .github/workflows/ci.yml | 5 ++++- README.md | 7 +++++++ pyproject.toml | 1 + tests/test_evaluation.py | 6 ++++-- tests/test_full_execution.py | 4 +++- tests/test_full_preview.py | 1 - tests/test_full_report.py | 6 ++++-- uv.lock | 31 +++++++++++++++++++++++++++++- vision_aid/evaluation/cli.py | 9 ++++++--- vision_aid/evaluation/full.py | 6 +++++- vision_aid/evaluation/llm_audit.py | 2 ++ vision_aid/evaluation/page.py | 2 +- 12 files changed, 67 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f753ada..263b872 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ permissions: jobs: checks: - name: Dependencies, imports, pipeline + name: Dependencies, lint, imports, pipeline runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -53,6 +53,9 @@ jobs: print('all entry points import cleanly') " + - name: Evaluator lint + run: uv run ruff check vision_aid/evaluation entry_points/model_evaluation.py tests + - name: Network-free test suite run: uv run pytest -q diff --git a/README.md b/README.md index 4271b2f..19ac658 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,7 @@ everything it does is free: - `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock - entry points import +- evaluator code, its entry point, and tests pass Ruff lint checks - the tracked synthetic pytest suite (`uv run pytest -q`), with no private data or provider calls - a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed - `index.html`'s inline JavaScript parses @@ -285,6 +286,12 @@ covers the evaluator end to end through its CLI with a fake provider. Optional detailed evaluator cases live in the ignored `.local-tests/evaluation/` folder; run them with `uv run pytest -q .local-tests/evaluation`. CI never needs them. +Run the evaluator lint checks locally with the dev dependencies installed by `uv sync`: + +```bash +uv run ruff check vision_aid/evaluation entry_points/model_evaluation.py tests +``` + ## Running the Pipeline ### Dry run (no API calls, no cost) diff --git a/pyproject.toml b/pyproject.toml index 4a5d662..165efac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ visionaid-evaluate = "vision_aid.evaluation.cli:main" [dependency-groups] dev = [ "pytest>=8.3.5", + "ruff>=0.17.0", ] [tool.pytest.ini_options] diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 2437293..9a04aac 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,9 +1,9 @@ """Run, review, and report Evaluation runs through the public CLI.""" import csv -from contextlib import nullcontext import hashlib import json +from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -247,7 +247,9 @@ def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( assert not (workspace / ".model-evaluation").exists() monkeypatch.setattr("builtins.input", lambda _: "no") for interactive in (False, True): - monkeypatch.setattr("sys.stdin", SimpleNamespace(isatty=lambda: interactive)) + monkeypatch.setattr( + "sys.stdin", SimpleNamespace(isatty=lambda interactive=interactive: interactive), + ) assert run_cli(workspace, "--model", model, "--live") == 0 runs = list((workspace / ".model-evaluation" / "runs").iterdir()) assert len(runs) == 2 diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py index fc73629..7b029f0 100644 --- a/tests/test_full_execution.py +++ b/tests/test_full_execution.py @@ -3,10 +3,12 @@ import json import pytest - from test_evaluation import recording_provider_usage from test_full_preview import URLS, full_workspace, only_full, preview +# Re-export the shared fixture for pytest discovery. +__all__ = ["full_workspace"] + def children(run, manifest): """Load each page's child directory and manifest in evaluation order.""" diff --git a/tests/test_full_preview.py b/tests/test_full_preview.py index d53ab34..0aa2d0a 100644 --- a/tests/test_full_preview.py +++ b/tests/test_full_preview.py @@ -3,7 +3,6 @@ import csv import hashlib import json -from pathlib import Path import openpyxl import pytest diff --git a/tests/test_full_report.py b/tests/test_full_report.py index e03b97f..7769902 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -3,17 +3,19 @@ import csv import hashlib import json -from pathlib import Path from types import SimpleNamespace import openpyxl import pytest - from test_evaluation import recording_provider_usage from test_full_execution import children from test_full_preview import URLS, full_workspace, only_full, preview + from vision_aid.evaluation.cli import main +# Re-export the shared fixture for pytest discovery. +__all__ = ["full_workspace"] + def save_json(path, value): """Save a deliberate evidence mutation or synthetic input.""" diff --git a/uv.lock b/uv.lock index ce76b0b..3600ffb 100644 --- a/uv.lock +++ b/uv.lock @@ -860,6 +860,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "ruff" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/a7/70debb024dfacda67b8e560cc7511f52b34b9348a8cc8c1ec23036dcd51d/ruff-0.17.0.tar.gz", hash = "sha256:5cd03240d8208a557c2a9655a5cb07ebe36aa6bb35065f97d48c1f6adef5a322", size = 5320554, upload-time = "2026-10-09T19:47:29.248Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f8/ee5ab9da6089eae2a33e6008b01deb1eda19992c1c8e10661e98cee1640f/ruff-0.17.0-py3-none-linux_armv6l.whl", hash = "sha256:0e271826af9a20d18c6cfae8c51e82959167c24859686ddd3eb9a7f0842ce81e", size = 10148907, upload-time = "2026-10-09T19:46:38.695Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d9/2f81fb5a9d580afbb11b1c8ff915233a11f2a1b27405d7991f183c5e1976/ruff-0.17.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5f0ca4a40f81403689c04f12966e22f44e329ae362072d8f1587b7bda87f603b", size = 10224208, upload-time = "2026-10-09T19:46:41.711Z" }, + { url = "https://files.pythonhosted.org/packages/a7/20/643f3c8f75594f937b2bf74801241c56a2e2b8e139d24dff8b66b28cdd7f/ruff-0.17.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cbf7149e0927dc3295d5d64679a4765576eef71b00782b2ae969ef82274d6bb9", size = 9273324, upload-time = "2026-10-09T19:46:44.323Z" }, + { url = "https://files.pythonhosted.org/packages/ec/91/627700b233d367736cb274f1bd0b47d1f2b12f68878192812bd875adadc3/ruff-0.17.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ee90156522998c3037059d8f66885c8adeeaf7643bdce2caceee196ecd23e0", size = 9953190, upload-time = "2026-10-09T19:46:47.155Z" }, + { url = "https://files.pythonhosted.org/packages/cd/92/91f7b5ed39490f89d6cbf56e1f543c383667a725efa8e2c2dee0f01f5591/ruff-0.17.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d8e4a002a94cd9d0dc48b51dc69d807a172b5b9bf2b668e656424dc5b55ead1", size = 9961746, upload-time = "2026-10-09T19:46:50.098Z" }, + { url = "https://files.pythonhosted.org/packages/87/c5/7310f9fc63ce11ff6394edbd5e85433dfb0e14c9fbf6ccc97f1538491bc7/ruff-0.17.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0b8a60c06a218c337e1161638d34757f83449243e2db161483ddf948e53ad14", size = 10625483, upload-time = "2026-10-09T19:46:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8d/97443f0dca4a03a0bc7629fd396fd494a1cb6666121e38c5075acb217d8f/ruff-0.17.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a330178bdffc4205dbf3bda11d93e059e388fd6546f8cdd304501a9160363c0d", size = 11350129, upload-time = "2026-10-09T19:46:56.486Z" }, + { url = "https://files.pythonhosted.org/packages/9c/0a/c525efd9777be4b6b012e6969a3012648468e7e6c4b3e5b46af69f46e8eb/ruff-0.17.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7bb08489e234876fa2da67ae3ea938e9a2156da80293e0e4365abd6973d98329", size = 10906223, upload-time = "2026-10-09T19:47:00.263Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3c/4a01195d93420cad1175bedad13a515dc8a56f95a6e39789b92e582682f5/ruff-0.17.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc73e7c133e82d55b5f15897b2a442d72c0cb4a0c886c46801ce3c247150b60c", size = 10386314, upload-time = "2026-10-09T19:47:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/c7/72/1a3951665485a921f6375f91e754a1854d5a645d41acc3642668064ff64d/ruff-0.17.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:db4f74c533403ab70fe4007873f6ae0c9f94a8b03158cf48d78788e47cdbe399", size = 10565464, upload-time = "2026-10-09T19:47:05.831Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/539b4d8c082f57e18db8ae2be85a460d77861c79dcd5798e32b536a6a06f/ruff-0.17.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3d8cc360e666d1914e47b0777c6906d70cf18891a55532bd0a16844195d70859", size = 10079468, upload-time = "2026-10-09T19:47:08.617Z" }, + { url = "https://files.pythonhosted.org/packages/e0/b8/84286966db79434e8c26b585b0a0f6897cb3ab1c51a4aa4df10c28488b62/ruff-0.17.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d66de796b726c4801e05fa99a2a8d7a780e107be222486c304ab61765561e866", size = 9958556, upload-time = "2026-10-09T19:47:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/69/50/27b6eed27b83fcdd5bfa0d52b83231e29094754374698da404d094487ae3/ruff-0.17.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c3f268baf004aea944f040623327119527ea231af15f7fb7890e82cea0679589", size = 10352173, upload-time = "2026-10-09T19:47:14.188Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/955399fd13044cd827862044117d784a59e3196f6cce7424908ac9a7f914/ruff-0.17.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:864b6c1acb6b0bccf94b5a3938a1531fd09aaca5e5659a2e7bf0f3cf2a685540", size = 10791761, upload-time = "2026-10-09T19:47:16.931Z" }, + { url = "https://files.pythonhosted.org/packages/ae/bf/024e01e1f6aec87768696725e648ed5b438941341eea8f8100beb681961f/ruff-0.17.0-py3-none-win32.whl", hash = "sha256:5e50aa5b84decd9fe5b0bb0e6f71c3b592f1767ed09faa4b7207d933961e35cd", size = 9904112, upload-time = "2026-10-09T19:47:19.75Z" }, + { url = "https://files.pythonhosted.org/packages/cc/77/1ee73df41dcc8d1cdb686ee4bc46ea29704ea175feb6b95c78420f631ab8/ruff-0.17.0-py3-none-win_amd64.whl", hash = "sha256:8ab76bcda86dfd28e13776cb5de3c7bcdcf1ae3d37ed761113d1a5a415dc134c", size = 10115585, upload-time = "2026-10-09T19:47:22.698Z" }, + { url = "https://files.pythonhosted.org/packages/fd/71/eb4f0ccc844aece56963e8578df9c95d4c00f580547d52329d4035d3af18/ruff-0.17.0-py3-none-win_arm64.whl", hash = "sha256:c154c73ff43f9854395e24cac507af13078962e53d2b511605058d22af1fdb88", size = 9872035, upload-time = "2026-10-09T19:47:26.306Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -947,6 +972,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "pytest" }, + { name = "ruff" }, ] [package.metadata] @@ -962,7 +988,10 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8.3.5" }] +dev = [ + { name = "pytest", specifier = ">=8.3.5" }, + { name = "ruff", specifier = ">=0.17.0" }, +] [[package]] name = "websockets" diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 95d42d4..219d94c 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -16,14 +16,17 @@ from dotenv import load_dotenv from vision_aid.evaluation.full import ( - capture_snapshots, execute_full_evaluation, load_full_inputs, - prepare_full_evaluation, report_full_evaluation, + capture_snapshots, + execute_full_evaluation, + load_full_inputs, + prepare_full_evaluation, + report_full_evaluation, ) from vision_aid.evaluation.human_audit import import_homepage_human_findings from vision_aid.evaluation.llm_audit import ( DEFAULT_MODEL, - LLMRequestConfig, MODEL_PRESETS, + LLMRequestConfig, evaluation_configuration, execute, positive_cost, diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py index 8aef3ce..c372be3 100644 --- a/vision_aid/evaluation/full.py +++ b/vision_aid/evaluation/full.py @@ -13,7 +13,11 @@ from vision_aid.evaluation.human_audit import import_all_human_findings, list_audit_urls from vision_aid.evaluation.llm_audit import ( - LLMRequestConfig, USAGE_KEYS, evaluation_configuration, execute, write_json, + USAGE_KEYS, + LLMRequestConfig, + evaluation_configuration, + execute, + write_json, ) from vision_aid.evaluation.page import PageReview, prepare_page, validated_page_review from vision_aid.evaluation.review import write_combined_report, write_report diff --git a/vision_aid/evaluation/llm_audit.py b/vision_aid/evaluation/llm_audit.py index a6d8b9d..0a8274b 100644 --- a/vision_aid/evaluation/llm_audit.py +++ b/vision_aid/evaluation/llm_audit.py @@ -10,6 +10,8 @@ from processing_scripts.llm_client.audit import ( AuditRequestClient as LLMRequestClient, +) +from processing_scripts.llm_client.audit import ( AuditRequestConfig as LLMRequestConfig, ) from vision_aid.evaluation.normalization import normalize_prompt_response diff --git a/vision_aid/evaluation/page.py b/vision_aid/evaluation/page.py index 69b1261..2ab0a1b 100644 --- a/vision_aid/evaluation/page.py +++ b/vision_aid/evaluation/page.py @@ -12,8 +12,8 @@ from entry_points.run_pipeline import run_pipeline from vision_aid.evaluation.llm_audit import ( - LLMRequestConfig, USAGE_KEYS, + LLMRequestConfig, evaluation_configuration, write_json, ) From 83af37799fa3a6191756fc46fb536f98b310d887 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:39:27 -0400 Subject: [PATCH 083/126] feat(evaluator): generate standalone single-page HTML reports --- README.md | 35 +++-- docs/evaluation-report-verification.md | 45 +++++++ pyproject.toml | 3 +- requirements.txt | 4 + tests/test_evaluation.py | 125 ++++++++++++++++-- tests/test_full_report.py | 4 +- uv.lock | 142 +++++++++++++++++++++ vision_aid/evaluation/cli.py | 2 +- vision_aid/evaluation/html_report.py | 141 ++++++++++++++++++++ vision_aid/evaluation/page.py | 9 +- vision_aid/evaluation/templates/page.html | 64 ++++++++++ vision_aid/evaluation/templates/report.css | 25 ++++ 12 files changed, 564 insertions(+), 35 deletions(-) create mode 100644 docs/evaluation-report-verification.md create mode 100644 vision_aid/evaluation/html_report.py create mode 100644 vision_aid/evaluation/templates/page.html create mode 100644 vision_aid/evaluation/templates/report.css diff --git a/README.md b/README.md index 19ac658..6243035 100644 --- a/README.md +++ b/README.md @@ -503,16 +503,22 @@ From the repository root, use the supplied `Pristine Accessibility Defect Report uv run visionaid-evaluate report --run-dir "$RUN_DIR" --reviewer "Your name or agent description" ``` - Open `$RUN_DIR/report.md`. It explains how to read the homepage evaluation, - then shows the model, run ID, reviewer, three detection rates, and estimated - cost with token counts and the pricing date. A table follows the Human audit's - original order and marks catches by each part, including cross-catches. The - remaining sections show what each part caught and why each match counts, - every missed finding with any recorded explanations, and findings not - testable from the page file with their reasons. IDs, locations, and raw JSON - stay in the run artifacts. Not-testable findings are left out of the LLM and - Programmatic rates but remain in the Overall denominator. Reporting rejects - incomplete runs, missing classifications, and invalid or reused finding IDs. + Open `$RUN_DIR/report.html` in a browser. This self-contained offline report + identifies the evaluated page, model, run ID, and reviewer, and leads with the + LLM detection rate. Programmatic and Overall detection rates show exact counts + and explain their denominators. Cost, recorded tokens, and saved pricing + provenance are included. + + The findings table follows the Human audit's original order. Each **View + evidence** control opens the original Human evidence, Eligibility classification, + explanation, matched finding text and IDs, check identity, rationale, and notes. + These native controls work with a keyboard and with JavaScript disabled. A copied + HTML file needs no neighboring run files or internet connection. + + Not-testable findings remain in Overall while staying outside source-specific + denominators. Reporting rejects incomplete runs, missing classifications, and + invalid or reused finding IDs. Successful regeneration replaces that report's + superseded Markdown file. It never repeats paid model calls. ### Capture the Human audit's listed pages @@ -712,7 +718,7 @@ non-empty (for tables, `issues` or `header_clarity_issues`). Every `placeholder_as_label` item is a finding, because that check only receives placeholder-only fields. Yes/no answers never create a finding on their own. A table response may be one object or a list. These findings have -`"screen_only": true`. The evaluation report, `report.md`, labels them as shown +`"screen_only": true`. The individual Evaluation report, `report.html`, labels them as shown on screen only and absent from va-dat's CSV report. ### Evaluation run files @@ -720,7 +726,8 @@ on screen only and absent from va-dat's CSV report. The run root holds the review evidence and scored results. Supporting files are together under `$RUN_DIR/artifacts/`. Without `--live`, the command prepares the prompts but makes no model requests; neither LLM output file exists yet. -Successful reporting adds `report.md` at the root. +The individual `report` command adds `report.html` at the root. The Full +evaluation `report-all` command continues to produce Markdown until its migration. New runs and reporting use only this layout. Existing run directories remain untouched and require manual reorganization before the updated `report` command @@ -735,7 +742,7 @@ compatibility fallback or automatic migration. | `normalized-llm-findings.json` | LLM findings with `finding_id` values for `review.csv`, read as described above. An Unreadable response can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | | `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | | `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | -| `report.md` | Scored results generated after successful reporting. | +| `report.html` | Self-contained individual Evaluation report generated after successful reporting. | | `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | | `artifacts/pricing.json` | The versioned public model rates and capture date used to estimate this run's cost from reported usage. Older runs retain their original price schedule. | | `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | @@ -754,7 +761,7 @@ evidence. Check check the file under `artifacts/prompts/` to see whether the model received the relevant evidence. Record decisions in `review.csv`; the `report` command reads that CSV and the -run evidence to write `report.md`. The pipeline briefly writes +run evidence to write `report.html`. The pipeline briefly writes `artifacts/manifest.json`, but the Evaluation command removes it and keeps the relevant metadata in `run.json`. diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md new file mode 100644 index 0000000..9e86548 --- /dev/null +++ b/docs/evaluation-report-verification.md @@ -0,0 +1,45 @@ +# Evaluation report verification + +## Individual HTML report, 2026-10-15 + +The individual `report` command was verified against synthetic completed runs +through the public CLI. Provider responses in the execution fixtures are replayed; +report regeneration reads saved evidence and makes no model requests. + +- Full suite: 137 tests passed, including existing Full evaluation reporting. +- `ruff check vision_aid/evaluation tests`: passed. Repository-wide Ruff also + reports 74 existing findings outside these directories; those files were not + changed for reporting. +- CLI coverage includes hand-calculated source and Overall rates, cross-catches, + zero denominators, empty findings, attribution, evidence ownership, invalid + decisions, canonical Human evidence despite edited CSV copies, jointly matched + findings, readable IDs/check provenance, code-like evidence, blank notes, and + failure on missing required rendering fields. +- Jinja2 uses HTML autoescaping and strict undefined values. A failed rendering + publishes no HTML and preserves a superseded Markdown report. Successful + generation atomically replaces HTML and removes that report's Markdown output. +- `uv build --wheel` included the Jinja template and CSS. Installing the wheel + into a fresh environment and invoking its `visionaid-evaluate report` from + `/tmp`, outside the checkout, generated the report successfully. + +A generated report containing six Human findings, a joint match, and hostile +HTML-like model/reviewer text was copied into a directory containing only +`report.html`. Checks used the local-file protocol with networking disabled: + +| Check | Actual coverage | Result | +| --- | --- | --- | +| Standalone resources | Chromium 153.0.8010.12, Linux, Playwright offline context | No HTTP requests; report readable | +| Automated accessibility | axe-core 4.11.1, WCAG 2 A/AA and 2.1/2.2 AA tags | No violations or incomplete checks in the tested initial document | +| Narrow layout | Chromium viewport 320 × 800 CSS pixels | Document and body width 320; findings table scrolls in its own region | +| Native disclosure | Chromium, JavaScript disabled, focus first summary and press Enter | Evidence opened; six disclosures were initially collapsed | + +The disposable copied report was saved at +`/tmp/va-dat-ticket01-standalone/report.html`; the installed-package smoke output +was saved under `/tmp/va-dat-ticket01-installed-run/`. These temporary artifacts +are not committed. + +Manual keyboard inspection, actual browser zoom, and screen-reader verification +remain outstanding. Headless keyboard automation and a narrow viewport do not +establish those manual results or WCAG conformance. The stakeholder's browser and +screen-reader setup is unknown. Combined HTML reports, charts, filtering, +search, and complete printing require later ticket verification. diff --git a/pyproject.toml b/pyproject.toml index 165efac..87b54ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "anthropic~=0.84.0", "beautifulsoup4>=4.9.2", "lxml>=4.9.0", + "jinja2>=3.1.6", "openai>=1.0.0", "google-genai>=1.0.0", "openpyxl>=3.1.5", @@ -37,5 +38,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.json"] +"vision_aid.evaluation" = ["pricing.json", "templates/*.html", "templates/*.css"] "processing_scripts.llm" = ["*.txt"] diff --git a/requirements.txt b/requirements.txt index d70272b..697adfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,12 +52,16 @@ idna==3.18 # anyio # httpx # requests +jinja2==3.1.6 + # via visionaid jiter==0.16.0 # via # anthropic # openai lxml==6.1.1 # via visionaid +markupsafe==3.0.4 + # via jinja2 openai==2.53.0 # via visionaid openpyxl==3.1.5 diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 9a04aac..4cb729c 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -9,6 +9,7 @@ import openpyxl import pytest +from bs4 import BeautifulSoup from processing_scripts.llm.registry import PROMPT_REGISTRY from vision_aid.evaluation import llm_audit @@ -76,9 +77,9 @@ def run_cli(workspace: Path, *extra: str) -> int: def report(run: Path) -> str: - """Score a reviewed run and return the Markdown report.""" + """Score a reviewed run and return readable HTML text.""" assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 - return (run / "report.md").read_text(encoding="utf-8") + return BeautifulSoup((run / "report.html").read_text(encoding="utf-8"), "html.parser").get_text(" ", strip=True) def refused(run: Path, capsys: pytest.CaptureFixture[str], reviewer: str = REVIEWER) -> str: @@ -86,7 +87,7 @@ def refused(run: Path, capsys: pytest.CaptureFixture[str], reviewer: str = REVIE with pytest.raises(SystemExit) as error: main(["report", "--run-dir", str(run), "--reviewer", reviewer]) assert error.value.code == 2 - assert not (run / "report.md").exists() + assert not (run / "report.html").exists() return capsys.readouterr().err @@ -593,6 +594,7 @@ def test_historical_reports_keep_saved_prices_and_stored_estimate(workspace, mon text = report(run) assert "Estimated run cost: $5.40" in text assert "published prices dated 2025-10-01" in text + assert "https://platform.claude.com/docs/en/about-claude/pricing" in text assert "180000 input tokens" in text and "18000 output tokens" in text assert all(path.read_bytes() == saved for path, saved in originals.items()) @@ -658,16 +660,19 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): "page's HTML, it caught 1 (50.0%)") in text assert "they caught 1 (50.0%)" in text assert "Of all 6 Human findings, either part of the tool caught 3 (50.0%)" in text - assert "| Skipped heading | 1.3.1 | Programmatic checks | ✓ | ✗ |" in text - assert "Why this counts: Full issue" in text - missed = text.split("## Missed\n", 1)[1].split("## Not testable", 1)[0] - assert "### No captions" in missed and "No full finding" in missed - excluded = text.split("## Not testable from the page file\n", 1)[1] - assert "### Hidden menu" in excluded and "Needs a browser" in excluded - assert "### Unclear claim" in excluded and "Needs judgment" in excluded + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + rows = document.select("#findings tbody > tr") + assert [cell.get_text(" ", strip=True) for cell in rows[2].find_all(["th", "td"], recursive=False)[:5]] == [ + "Skipped heading", "1.3.1", "Programmatic checks", "Caught", "Not caught", + ] + assert "Full issue" in rows[0].details.get_text() + assert "No full finding" in rows[5].details.get_text() + assert "Needs a browser" in rows[3].details.get_text() + assert "Needs judgment" in rows[4].details.get_text() assert ("1 of 18 LLM checks returned an Unreadable response that the tool " "would skip: page title check.") in text - assert all(finding_id not in text for finding_id in [*llm, programmatic]) + assert llm[0] in text and llm[1] in text and programmatic in text + assert llm[2] not in text edit_review(run, *[{"classification": "ambiguous", "llm_finding_id": "", "programmatic_finding_id": "", "match_reason": ""}] * 6) text = report(run) @@ -802,5 +807,101 @@ def ok(response: object) -> dict: text = report(run) assert ("LLM findings are read from each response with the tool's own rules, " "before its false-positive filter and deduplication.") in text - assert ("- LLM (media captions check; shown on screen only, not in the tool's " + assert ("LLM (media captions check; shown on screen only, not in the tool's " "CSV report): No captions track") in text + + +def test_single_page_report_is_standalone_html(workspace, monkeypatch, capsys): + """A completed review produces readable offline HTML with native evidence controls.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + legacy = run / "report.md" + legacy.write_text("Superseded report") + assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert f"HTML report: {run / 'report.html'}" in capsys.readouterr().out + assert not legacy.exists() + assert document.html["lang"] == "en" + assert document.title and document.h1 + assert document.select_one('a[href="#main"]') and document.main["id"] == "main" + assert document.style and not document.select("script[src], link[rel=stylesheet], img[src]") + table = document.find("table", attrs={"id": "findings"}) + assert table.caption and len(table.tbody.find_all("tr", recursive=False)) == 6 + assert [row.th.get_text(strip=True) for row in table.tbody.find_all("tr", recursive=False)] == [ + "Unclear link", "Missing destination", "Skipped heading", "Hidden menu", "Unclear claim", "No captions", + ] + disclosures = table.find_all("details") + assert len(disclosures) == 6 and all(not item.has_attr("open") for item in disclosures) + assert all("View evidence" in item.summary.get_text() for item in disclosures) + assert "Vague link\nfor everyone" in document.get_text() + + +def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monkeypatch): + """Editable evidence cannot replace originals, and joint matches retain check provenance.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + findings = load(run, "normalized-llm-findings.json") + hostile = '' + findings[0].update(problem=hostile, element="", location="Footer", checklist="Synthetic checklist 42") + (run / "normalized-llm-findings.json").write_text(json.dumps(findings)) + edit_review(run, { + "classification": "llm_eligible", "llm_finding_id": ";".join(item["finding_id"] for item in findings[:2]), + "match_reason": "Together they fully cover the link", "review_notes": hostile, + "actual result": "Editable evidence replacement", "Issue Title": "Edited title", + }, *[{"classification": "ambiguous"}] * 5) + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: pytest.fail("report made a paid request")) + assert main(["report", "--run-dir", str(run), "--reviewer", hostile]) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + evidence = document.select_one("#findings tbody tr details") + text = evidence.get_text(" ", strip=True) + assert "Editable evidence replacement" not in text and "Edited title" not in text + assert "Vague link\nfor everyone" in text and "Unclear link" in text + assert hostile in text and not document.select("script, img, [onerror]") + assert "Synthetic checklist 42" in text + assert all(item["finding_id"] in text for item in findings[:2]) + assert "Together they fully cover the link" in text and "Footer" in text + assert "" in text and findings[0]["page_url"] in text + + +@pytest.mark.parametrize("missing", ["human_audit_filename", "configuration.model"]) +def test_report_requires_complete_rendering_data(workspace, monkeypatch, capsys, missing): + """Absent required fields cannot publish HTML or discard a legacy report.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + edit_review(run, *[{"classification": "llm_eligible"}] * 6) + manifest = load(run, "run.json") + if missing == "configuration.model": + del manifest["configuration"]["model"] + else: + del manifest[missing] + (run / "run.json").write_text(json.dumps(manifest)) + legacy = run / "report.md" + legacy.write_text("Previous report") + assert "Cannot render complete report" in refused(run, capsys) + assert legacy.read_text() == "Previous report" + assert not list(run.glob(".report.html.*.tmp")) + + +def test_report_with_no_human_findings_has_no_false_zero_rates(workspace, monkeypatch): + """An empty saved review explains absent denominators in words.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + audit = load(run, "artifacts/human-findings.json") + audit["human_findings"] = [] + (run / "artifacts/human-findings.json").write_text(json.dumps(audit)) + with (run / "review.csv").open() as stream: + header = next(csv.reader(stream)) + with (run / "review.csv").open("w", newline="") as stream: + csv.writer(stream).writerow(header) + text = report(run) + assert "No Human findings were imported for this page" in text + assert "No Human findings were expected to be detected by the LLM" in text + assert "No Human findings were expected to be detected by the Programmatic checks" in text + assert "(0.0%)" not in text + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert not document.select("#findings tbody tr") diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 7769902..8fbc1ba 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -308,9 +308,9 @@ def test_completed_child_report_survives_later_parent_failure(full_workspace, mo row.update(classification="llm_eligible", classification_reason="Saved evidence") save_review(first, rows) assert main(["report", "--run-dir", str(first), "--reviewer", "Page reviewer"]) == 0 - saved = (first / "report.md").read_bytes() + saved = (first / "report.html").read_bytes() assert "Incomplete Full evaluations" in refused_report(run, capsys) - assert (first / "report.md").read_bytes() == saved + assert (first / "report.html").read_bytes() == saved def test_reviewer_description_is_required(completed_review, capsys): diff --git a/uv.lock b/uv.lock index 3600ffb..659ddb4 100644 --- a/uv.lock +++ b/uv.lock @@ -427,6 +427,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.16.0" @@ -615,6 +627,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/55/18dbb4778b30ada5ce071608503cc3edc9e14e13d868c17a6d178fc30f7a/markupsafe-3.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346", size = 11520, upload-time = "2026-10-02T23:04:36.299Z" }, + { url = "https://files.pythonhosted.org/packages/6c/14/0b05f79b4733e264a18d08fe08fa1df7347630ff32a6cb82180d9dccec55/markupsafe-3.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91", size = 12063, upload-time = "2026-10-02T23:04:37.291Z" }, + { url = "https://files.pythonhosted.org/packages/ca/3a/63ba10b6c1463216b3e4df669a9f0e5a3b0c3071557d2e8229e3968c79fb/markupsafe-3.0.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef", size = 24321, upload-time = "2026-10-02T23:04:38.262Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/5f015261b76ad633d187ef6f388b413aedd64a8773c4df59e530a0be5525/markupsafe-3.0.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169", size = 28845, upload-time = "2026-10-02T23:04:39.295Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/26e594b26be40c2f1fec63ccf8a8b99d0335a5b2cbe84835c7d82a994375/markupsafe-3.0.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb", size = 24701, upload-time = "2026-10-02T23:04:40.348Z" }, + { url = "https://files.pythonhosted.org/packages/81/a5/a513b76c139a3915b43404324e55c0b7979ae4f0d39eb6f075b0282e90a8/markupsafe-3.0.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808", size = 22961, upload-time = "2026-10-02T23:04:41.372Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/4c66192c100b4542bcbe392ae06696b670f66927be3ac38a213234778ff9/markupsafe-3.0.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692", size = 21920, upload-time = "2026-10-02T23:04:42.429Z" }, + { url = "https://files.pythonhosted.org/packages/cb/17/ac3662678bfbad649893117ada2ba44dc30bf56884e84f13154a792b10f1/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d", size = 23726, upload-time = "2026-10-02T23:04:43.531Z" }, + { url = "https://files.pythonhosted.org/packages/f7/af/fe47cee339180a69ebca3c57fb3483d0f5cbd1e8337d1871fb1d9c1aebee/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21", size = 22539, upload-time = "2026-10-02T23:04:44.618Z" }, + { url = "https://files.pythonhosted.org/packages/12/32/d55440ba140442800e02d799c9cb5ab597bf6ebdb1177b5ea39a11f797bd/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707", size = 23963, upload-time = "2026-10-02T23:04:45.656Z" }, + { url = "https://files.pythonhosted.org/packages/50/9d/9c86042cb364c2ad4c971e6d1247929effd25f714cd7ee11b05e6316445b/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e", size = 21506, upload-time = "2026-10-02T23:04:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/75/ef/5b824f03ba40c3b3652b6272d083d2fc4fcdd440de519a3a39ba2c3e7262/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7", size = 22955, upload-time = "2026-10-02T23:04:47.698Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e8/44cfcb5ea40e5e43cec7793ef90704ed0c475280839076c4345757eb8e59/markupsafe-3.0.4-cp311-cp311-win32.whl", hash = "sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5", size = 14094, upload-time = "2026-10-02T23:04:48.788Z" }, + { url = "https://files.pythonhosted.org/packages/91/89/f2b509f7bf79352e40117824c1070dbeafd4df67031d3fa98165a3134228/markupsafe-3.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3", size = 14328, upload-time = "2026-10-02T23:04:49.801Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5a/ccf22672a0f64dc682306e288f0dabcb06c7a201f3bb6e6cbf86d9e8ad03/markupsafe-3.0.4-cp311-cp311-win_arm64.whl", hash = "sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e", size = 14188, upload-time = "2026-10-02T23:04:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c3/a944f3b0df22bd129e96915b9f4e98d2eeca6516687d7618304a966c3c74/markupsafe-3.0.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed", size = 12787, upload-time = "2026-10-02T23:05:30.971Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d6/a44863f69d88b6c7e27889108f70d47aed259edf89d5df3c5fca1eac87d6/markupsafe-3.0.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59", size = 12525, upload-time = "2026-10-02T23:05:32.263Z" }, + { url = "https://files.pythonhosted.org/packages/17/8f/168ba80e532dd6a93f96f8f706f1ad41d7990b6e1aeedc1cc0d211a33497/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453", size = 11589, upload-time = "2026-10-02T23:05:33.251Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/aa2c95a574d3af39403a469b295886eb9b6d448da568cbebb5a2cbfdc2e5/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b", size = 11801, upload-time = "2026-10-02T23:05:34.315Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/34b810107d83840e768bf485de795893ebbae35b26ab061b487adfa0a692/markupsafe-3.0.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6", size = 11447, upload-time = "2026-10-02T23:05:35.302Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ab/2f8488f0f817a39fca068d2b17daf446bf5cdb3eae28c3720af534d873b4/markupsafe-3.0.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634", size = 11722, upload-time = "2026-10-02T23:05:36.363Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/e2d117b048d47282ade906fbfd92814cbee5647afc13fda88a3406039372/markupsafe-3.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f", size = 12061, upload-time = "2026-10-02T23:05:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a8/73a81135e85ba66217f5af7facb03bbb386807e1a729ab64532e4c802652/markupsafe-3.0.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9", size = 24434, upload-time = "2026-10-02T23:05:38.407Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ca/fa9216dd01efee2dfdacafe7df32b4d0170fbac694b0c258a193d6e53999/markupsafe-3.0.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f", size = 27183, upload-time = "2026-10-02T23:05:39.581Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4e/a469509e538d37af51103b17b073126973f2b1cbf197ff32c7ddf025cfe5/markupsafe-3.0.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c", size = 24813, upload-time = "2026-10-02T23:05:40.671Z" }, + { url = "https://files.pythonhosted.org/packages/8f/db/d7282caf7ab03af44d5d6fdbaa019b35c7d7f1c90588b839c07cba640d6a/markupsafe-3.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300", size = 23049, upload-time = "2026-10-02T23:05:41.864Z" }, + { url = "https://files.pythonhosted.org/packages/30/f3/b6a425206e6964efda6acee544d0eb01d1501784d0b8e2dcc74986f33b17/markupsafe-3.0.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0", size = 22088, upload-time = "2026-10-02T23:05:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8a/84d3582fc1f0d5bd466cdf2eebf175e172158a6e70701aacec1de1b35430/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977", size = 23846, upload-time = "2026-10-02T23:05:44.098Z" }, + { url = "https://files.pythonhosted.org/packages/1c/65/db101cce51b7ba4864ac491a9859d297dd1adf0e55b103fee9db9c47c527/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7", size = 22674, upload-time = "2026-10-02T23:05:45.23Z" }, + { url = "https://files.pythonhosted.org/packages/e0/49/ddee9813d71db0c7a5c9d97c832125e6758a0c844777f1cf076569bb0e22/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17", size = 24104, upload-time = "2026-10-02T23:05:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0e/7d8518d726726870a2399d69fd30d0fa36c5e57a2132c336b58d7c491073/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c", size = 21637, upload-time = "2026-10-02T23:05:47.48Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b0/b505e8a361ba557dbf3b3aa7331ea39b00d2022a26e925ff8463b9714bb3/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4", size = 23081, upload-time = "2026-10-02T23:05:48.611Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/9cc3cea873f980c75cbdb6f4277ce30ee955de38be0b3d02f14c108e0698/markupsafe-3.0.4-cp314-cp314-win32.whl", hash = "sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c", size = 14272, upload-time = "2026-10-02T23:05:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/80/f0/5792ff768a410f93ee3f84fc19345295ffc352d2c936b424cb37e514714c/markupsafe-3.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe", size = 14501, upload-time = "2026-10-02T23:05:50.788Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cf/3d074a8edffcc6899355232ff2543ae8d929733239596423b7db79698bc9/markupsafe-3.0.4-cp314-cp314-win_arm64.whl", hash = "sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a", size = 14364, upload-time = "2026-10-02T23:05:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/d9/31/87ce42159aae2163cf3bbbd0c44bc87780510eecab1ea3859099aed95dcb/markupsafe-3.0.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2", size = 11759, upload-time = "2026-10-02T23:05:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/5f/53/b047207eeb7752e960aca3eb1df5fb7eefa7dd4c62ac49bb156456c8a702/markupsafe-3.0.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977", size = 12096, upload-time = "2026-10-02T23:05:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/4326c88a13c7b755657d44b4bb986f8c3d9843ecba7e22d98661d87f9a57/markupsafe-3.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289", size = 25699, upload-time = "2026-10-02T23:05:55.15Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/990581b7474bfcf2cf34bed6ba5ea23bd87adb9d671213d68e88620e7a6b/markupsafe-3.0.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe", size = 27804, upload-time = "2026-10-02T23:05:56.29Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/89491878c28e8291f5aa2fffe2c2d57230d10ae366d55dd810b840513d78/markupsafe-3.0.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a", size = 25912, upload-time = "2026-10-02T23:05:57.416Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/680998b54efdea06fc114565cd739b6d059f826a0279219b218dfa750d29/markupsafe-3.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733", size = 23623, upload-time = "2026-10-02T23:05:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/ae/75/2709f5ac5de9467b40b10e2bb8f89cc63dfb74582e09aa734b1124a217de/markupsafe-3.0.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34", size = 23301, upload-time = "2026-10-02T23:05:59.94Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c8/39eadc6c5b14c9c7679bfb98f4d4c6a97863b5beb91839aca4d2d6e16e55/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978", size = 24950, upload-time = "2026-10-02T23:06:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5e/01037f8a43e8ccb0bffb4fbdc5212db05bf080fdd7286cd392332d58128a/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc", size = 23330, upload-time = "2026-10-02T23:06:02.441Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f4/23e83ce0596bb0cbe670502d31df8f757bbd01a392aa486fa3b40d1ed399/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc", size = 25239, upload-time = "2026-10-02T23:06:03.579Z" }, + { url = "https://files.pythonhosted.org/packages/88/5b/3708897368073cc683d524750474f41a77d2986152c380dcc55b20fdf340/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932", size = 22767, upload-time = "2026-10-02T23:06:04.699Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/ebda1307864b409e6b3115757a3d4a09cca46cfb6cc65191b5de226b424b/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6", size = 23668, upload-time = "2026-10-02T23:06:05.9Z" }, + { url = "https://files.pythonhosted.org/packages/09/15/98075cceac3b5ba0dbb8e4762a847be967d2befc349a2cf2d0ac77f62c9d/markupsafe-3.0.4-cp314-cp314t-win32.whl", hash = "sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691", size = 14301, upload-time = "2026-10-02T23:06:07.109Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/768b560fcc4156685cb563d922b217810cfa7bc135773367f62f1f9d2078/markupsafe-3.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464", size = 14526, upload-time = "2026-10-02T23:06:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/63/da554b4c97a6b0ea3229ca7fe8cbfb620be81613d517f482e85958550537/markupsafe-3.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c", size = 14421, upload-time = "2026-10-02T23:06:09.402Z" }, + { url = "https://files.pythonhosted.org/packages/a9/30/54d11c8ca027114898cab97421fb39e4ffd9ddf47cdbc44df2ec76722da9/markupsafe-3.0.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65", size = 12791, upload-time = "2026-10-02T23:06:10.485Z" }, + { url = "https://files.pythonhosted.org/packages/10/6d/97c913e253a14bd3cd0e15a5c56d13203b823fa7ee32498342896a072dc4/markupsafe-3.0.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163", size = 12527, upload-time = "2026-10-02T23:06:11.834Z" }, + { url = "https://files.pythonhosted.org/packages/26/f9/b86d032042a4d597d9e1997f0e5f63a3eedaf11258e0a05760b0a0a826ea/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92", size = 11602, upload-time = "2026-10-02T23:06:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dc/73c14c1eedf0ac5fa3292ba43435e6c49d2c2050f33cebde541f8f4807f1/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a", size = 11817, upload-time = "2026-10-02T23:06:14.227Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/2c2fcaa5fcee22d72c7819c0d536fd181c74a688e6143845419579cd2863/markupsafe-3.0.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429", size = 11446, upload-time = "2026-10-02T23:06:15.574Z" }, + { url = "https://files.pythonhosted.org/packages/88/54/9e5ec76c62e6e2834d5a93623018c943e8b3bb41d663e3fd4c03303b9b85/markupsafe-3.0.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8", size = 11719, upload-time = "2026-10-02T23:06:16.701Z" }, + { url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97", size = 12075, upload-time = "2026-10-02T23:06:17.855Z" }, + { url = "https://files.pythonhosted.org/packages/aa/85/b64fdb1f304848518742136983c24e96d967bfb59a0ea160e92736901ab0/markupsafe-3.0.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b", size = 24656, upload-time = "2026-10-02T23:06:18.963Z" }, + { url = "https://files.pythonhosted.org/packages/9c/18/23997d4c65b355da6390d61cd56e0ab3befd6ba8dda25cb40c602bd0fa6b/markupsafe-3.0.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9", size = 27737, upload-time = "2026-10-02T23:06:20.117Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/35998dead3c6af88c38265a56e58100211f036234ab88eb2283fd4cbce44/markupsafe-3.0.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653", size = 25003, upload-time = "2026-10-02T23:06:21.284Z" }, + { url = "https://files.pythonhosted.org/packages/82/96/ef49135ce260db4ca4a12b119ed468449cd248db6b1468e2112b546d7a2e/markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369", size = 23111, upload-time = "2026-10-02T23:06:22.524Z" }, + { url = "https://files.pythonhosted.org/packages/50/7d/83126e338bd88c17a220668235368ad719fd4638e426739858cbb8508f77/markupsafe-3.0.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19", size = 22403, upload-time = "2026-10-02T23:06:23.785Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/daf7e420de23c8206c365204e7b85e1251d8e19d34196a56336f316e5ed2/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e", size = 24083, upload-time = "2026-10-02T23:06:25.037Z" }, + { url = "https://files.pythonhosted.org/packages/19/3c/11eecdc06bc44ad5570350085b572ebf049e8f9a38d1ece6d76640b739cd/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811", size = 22931, upload-time = "2026-10-02T23:06:26.328Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/ac0fd77f2a726e56ecc3ca0235d095feace1358d1b822406c2a2ef26a4dc/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea", size = 24299, upload-time = "2026-10-02T23:06:27.742Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/c6bd842ad58ff5b3bc76eeed7e9a42a6f11adc5d090ec697b72c9672731e/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916", size = 21948, upload-time = "2026-10-02T23:06:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/a3/46/82f586711fed61e86faa1ee1bc317d68cd45a10c8bdbe3f7d1fdf9026ad8/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741", size = 23150, upload-time = "2026-10-02T23:06:30.583Z" }, + { url = "https://files.pythonhosted.org/packages/19/2d/2dfdce99318abbfa26925195fbc17db188c46a1ec6457be121b6f9cfeb42/markupsafe-3.0.4-cp315-cp315-win32.whl", hash = "sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b", size = 14269, upload-time = "2026-10-02T23:06:31.949Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl", hash = "sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214", size = 14496, upload-time = "2026-10-02T23:06:33.258Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/e73bd5016421d5d6e2fb6de7dd609f9de020942ac8c626526bd8c6eeaf82/markupsafe-3.0.4-cp315-cp315-win_arm64.whl", hash = "sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67", size = 14361, upload-time = "2026-10-02T23:06:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/90/df/cb8c3dc98d313a951df2f8968f44e4cb5643df6d3cab749a530ce2f7d972/markupsafe-3.0.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad", size = 11760, upload-time = "2026-10-02T23:06:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/d6/bb/4af9b3ca0753d654ac75f9531d5bd741bb77ca6e696f36807c475ffc099a/markupsafe-3.0.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99", size = 12108, upload-time = "2026-10-02T23:06:37.089Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d4/b56429313aee5fd59b079c3df5615299959e25e7113eb6d8caadbdd7d38a/markupsafe-3.0.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002", size = 25762, upload-time = "2026-10-02T23:06:38.419Z" }, + { url = "https://files.pythonhosted.org/packages/65/f5/34c181e891aa4f7d59c918584672e0c5eb7fffe76c1387d1246008bf4081/markupsafe-3.0.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e", size = 28400, upload-time = "2026-10-02T23:06:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/ad14694fd0ac9a5ce30bc6498f2999378f418583dd1679cca5a1b512957e/markupsafe-3.0.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c", size = 25946, upload-time = "2026-10-02T23:06:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a8/26b606445387d0ceb1eb1f21840094b84e4e3c3c3983d80d10b89823b490/markupsafe-3.0.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8", size = 23634, upload-time = "2026-10-02T23:06:42.748Z" }, + { url = "https://files.pythonhosted.org/packages/39/a2/b8814de672f1f0094d498bf646f2fec9d6356b503d28ef500b71c5095377/markupsafe-3.0.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe", size = 23488, upload-time = "2026-10-02T23:06:44.176Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/287223376fb73335a3cc5d6eb22c6ab01358cf33945a9c39c06b9dac3f4b/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2", size = 25037, upload-time = "2026-10-02T23:06:45.646Z" }, + { url = "https://files.pythonhosted.org/packages/f9/29/4df8355e313426d19e62ba33e0253c009ca12a0894ee77d67fa67255361c/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38", size = 23543, upload-time = "2026-10-02T23:06:47.264Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/8377731e8495668dcc768f645e717df18318c841edaf023a99395f6da9b4/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494", size = 25282, upload-time = "2026-10-02T23:06:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5f/373456e37ceb1478d657d6fe769cbe0a39f0a8dfc1548eeb19c471eefdd9/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d", size = 22993, upload-time = "2026-10-02T23:06:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/d7/93/2cbd5628435afb6f541bbaced4bce0c2edac4b09a142e6e928b8b0da9858/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894", size = 23712, upload-time = "2026-10-02T23:06:51.759Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/157e10966b033b363aeda5263e82596ee232a0b1d082fdbf90aa417ff083/markupsafe-3.0.4-cp315-cp315t-win32.whl", hash = "sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78", size = 14300, upload-time = "2026-10-02T23:06:53.241Z" }, + { url = "https://files.pythonhosted.org/packages/33/05/55884815414c9706a23deca150b72c25a62109e65b0b6ce232077802c719/markupsafe-3.0.4-cp315-cp315t-win_amd64.whl", hash = "sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c", size = 14522, upload-time = "2026-10-02T23:06:54.729Z" }, + { url = "https://files.pythonhosted.org/packages/92/f9/ecbde7149e95b8a0f18e16d5d747f7dc06049d5da2e4f77f6f5e4a1f46a8/markupsafe-3.0.4-cp315-cp315t-win_arm64.whl", hash = "sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba", size = 14417, upload-time = "2026-10-02T23:06:56.246Z" }, +] + [[package]] name = "openai" version = "2.53.0" @@ -962,6 +1102,7 @@ dependencies = [ { name = "anthropic" }, { name = "beautifulsoup4" }, { name = "google-genai" }, + { name = "jinja2" }, { name = "lxml" }, { name = "openai" }, { name = "openpyxl" }, @@ -980,6 +1121,7 @@ requires-dist = [ { name = "anthropic", specifier = "~=0.84.0" }, { name = "beautifulsoup4", specifier = ">=4.9.2" }, { name = "google-genai", specifier = ">=1.0.0" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "lxml", specifier = ">=4.9.0" }, { name = "openai", specifier = ">=1.0.0" }, { name = "openpyxl", specifier = ">=3.1.5" }, diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index 219d94c..c27d4c1 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -287,7 +287,7 @@ def _report(args: argparse.Namespace) -> int: """Validate human decisions and report a completed run.""" run = args.run_dir.resolve() report_page(run, args.reviewer) - print(f"Markdown report: {run / 'report.md'}") + print(f"HTML report: {run / 'report.html'}") return 0 diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py new file mode 100644 index 0000000..6d2baa0 --- /dev/null +++ b/vision_aid/evaluation/html_report.py @@ -0,0 +1,141 @@ +"""Prepare reader-facing data and publish self-contained evaluation reports.""" + +from __future__ import annotations + +import os +import tempfile +from decimal import Decimal +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from jinja2 import ( + Environment, + PackageLoader, + StrictUndefined, + TemplateError, + select_autoescape, +) + +from vision_aid.evaluation.review import _rate, _usage_description, detection_counts + +if TYPE_CHECKING: + from vision_aid.evaluation.page import PageReview + +EXPECTED = { + "llm_eligible": "LLM", + "programmatic": "Programmatic checks", + "unavailable_evidence": "Not testable from the page file", + "ambiguous": "Not testable from the page file", +} + + +def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: + """Resolve validated evidence and deterministic scores before rendering.""" + manifest = review.manifest + findings = {item["finding_id"]: item for item in review.llm + review.programmatic} + counts = detection_counts(review.rows) + rows = [] + for row in review.rows: + human, decision = row["human_finding"], row["decision"] + classification = decision["classification"].strip() + matched = [] + for kind in ("llm", "programmatic"): + for identity in row["matches"][kind]: + finding = findings[identity] + check = finding["prompt"].replace("_", " ") + " check" + if finding.get("screen_only"): + check += "; shown on screen only, not in the tool's CSV report" + matched.append( + { + "id": finding["finding_id"], + "source": "LLM" if kind == "llm" else "Programmatic checks", + "check": check, + "check_id": finding["prompt"], + "checklist": finding["checklist"], + "problem": finding["problem"], + "page_url": finding["page_url"], + "run_id": finding["run_id"], + "element": finding["element"], + "location": finding["location"], + } + ) + rows.append( + { + "human": human, + "decision": decision, + "expected": EXPECTED[classification], + "llm_outcome": "Caught" if row["matches"]["llm"] else "Not caught", + "programmatic_outcome": "Caught" + if row["matches"]["programmatic"] + else "Not caught", + "matched": matched, + } + ) + rates = [ + { + "label": label.removesuffix(" checks"), + **counts[kind], + "description": _rate( + label, counts[kind]["caught"], counts[kind]["eligible"] + ), + } + for kind, label in ( + ("llm", "LLM"), + ("programmatic", "Programmatic checks"), + ("overall", "Overall"), + ) + ] + return { + "title": "LLM accessibility evaluation", + "manifest": manifest, + "reviewer": reviewer, + "rows": rows, + "rates": rates, + "cost": f"{Decimal(manifest['estimated_cost_usd']):.2f}", + "pricing_description": review.schedule.price_description, + "pricing_identity": review.schedule.identity, + "pricing_version": review.schedule.version, + "pricing_sources": review.schedule.data.get( + "sources", + [review.schedule.data["source"]] + if "source" in review.schedule.data + else [], + ), + "usage_description": _usage_description(manifest["usage"]), + "unreadable": len(manifest["format_failures"]), + "prompts": len(manifest["prompts"]), + "failed_checks": ", ".join( + name.replace("_", " ") + " check" for name in manifest["format_failures"] + ), + } + + +def write_page_report(path: Path, review: PageReview, reviewer: str) -> None: + """Render completely, then atomically replace the report in its directory.""" + environment = Environment( + loader=PackageLoader("vision_aid.evaluation", "templates"), + autoescape=select_autoescape(("html",)), + undefined=StrictUndefined, + ) + try: + content = environment.get_template("page.html").render( + **page_data(review, reviewer) + ) + except TemplateError as error: + raise ValueError(f"Cannot render complete report: {error}") from error + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(content) + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/vision_aid/evaluation/page.py b/vision_aid/evaluation/page.py index 2ab0a1b..a81fd9f 100644 --- a/vision_aid/evaluation/page.py +++ b/vision_aid/evaluation/page.py @@ -11,6 +11,7 @@ from typing import Any from entry_points.run_pipeline import run_pipeline +from vision_aid.evaluation.html_report import write_page_report from vision_aid.evaluation.llm_audit import ( USAGE_KEYS, LLMRequestConfig, @@ -19,7 +20,7 @@ ) from vision_aid.evaluation.normalization import normalize_programmatic_findings from vision_aid.evaluation.pricing import DEFAULT_SCHEDULE, PriceSchedule -from vision_aid.evaluation.review import create_review, reviewed_rows, write_report +from vision_aid.evaluation.review import create_review, reviewed_rows from vision_aid.evaluation.schemas import HumanAudit @@ -144,8 +145,6 @@ def report_page(run: Path, reviewer: str) -> PageReview: if not reviewer.strip(): raise ValueError("A reviewer description is required") review = validated_page_review(run) - write_report( - run / "report.md", review.manifest, review.rows, review.llm, - review.programmatic, reviewer, review.schedule.price_description, - ) + write_page_report(run / "report.html", review, reviewer) + (run / "report.md").unlink(missing_ok=True) return review diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html new file mode 100644 index 0000000..ff37e1a --- /dev/null +++ b/vision_aid/evaluation/templates/page.html @@ -0,0 +1,64 @@ + + + + + +{{ title }}: {{ manifest.source_url }} + + + + +
+

{{ title }}

+

This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.

+

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

+

Review decisions by: {{ reviewer }}

+ +
+
+
+

Detection results

+{% for rate in rates %}

{{ rate.description }}

{% endfor %} +

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each Human finding once out of all imported Human findings.

+
+
+

Human findings

+

Evidence starts collapsed. Open any View evidence control to read the original Human finding and the review decision.

+
+ + + + +{% for row in rows %} + + + + +{% endfor %} +
Human findings and detection outcomes
Issue titleWCAGExpected sourceLLMProgrammatic checksEvidence
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }} +
+

Original Human finding

+
Human finding ID
{{ row.human.human_finding_id }}
Source
{{ row.human.source_sheet }}, row {{ row.human.source_row }}; scope: {{ row.human.page_scope }}; element: {{ row.human.source_element }}
+{% for label, value in row.human.raw_evidence.items() %}
{{ label }}
{{ value }}
{% endfor %}
+

Review decision

+
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
Review notes
{{ row.decision.review_notes }}
+{% for finding in row.matched %}

{{ finding.source }} matching evidence

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
Page provenance
{{ finding.page_url }}; run {{ finding.run_id }}
Element
{{ finding.element }}
Location
{{ finding.location }}
{% endfor %} +
+{% if not rows %}

No Human findings were imported for this page.

{% endif %} +
+

How to read this report

+

The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.

+

LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.

+

LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.

+

The tool saw only saved HTML, not a live browser or screen reader. A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.

+

Not testable from the page file: These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator. Their recorded explanations appear in the evidence.

+
+

Cost and provenance

+

Estimated run cost: ${{ cost }}, calculated using provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

+

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

+{% if unreadable %}

{{ unreadable }} of {{ prompts }} LLM checks returned an Unreadable response that the tool would skip: {{ failed_checks }}.

{% endif %} +
Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
Benchmark snapshot SHA-256
{{ manifest.snapshot_sha256 }}
Saved pricing version
{{ pricing_version }}
Saved pricing SHA-256
{{ pricing_identity }}
{% for source in pricing_sources %}
Pricing source address
{{ source }}
{% endfor %}
+
+
+ + diff --git a/vision_aid/evaluation/templates/report.css b/vision_aid/evaluation/templates/report.css new file mode 100644 index 0000000..de7b90c --- /dev/null +++ b/vision_aid/evaluation/templates/report.css @@ -0,0 +1,25 @@ +:root { color-scheme: light; font-family: system-ui, sans-serif; color: #182838; background: #fff; line-height: 1.6; } +* { box-sizing: border-box; } +body { margin: 0 auto; max-width: 90rem; padding: 1.25rem; } +h1, h2, h3 { line-height: 1.25; } +h1 { font-size: clamp(1.8rem, 5vw, 2.7rem); } +a { color: #164e88; text-underline-offset: .18em; } +a:focus-visible, summary:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible { outline: 3px solid #8c3500; outline-offset: 4px; } +.skip { position: absolute; top: -10rem; padding: .7rem; background: #fff; } +.skip:focus { top: .5rem; } +nav { display: flex; flex-wrap: wrap; gap: 1.25rem; border-block: 1px solid #8090a0; padding-block: .8rem; } +section { margin-block: 2rem; } +.table-scroll { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; margin-block: 1rem; } +caption { font-weight: 700; text-align: left; margin-bottom: .5rem; } +th, td { text-align: left; vertical-align: top; border: 1px solid #8a99a6; padding: .65rem; overflow-wrap: anywhere; } +thead { background: #edf2f6; } +summary { cursor: pointer; color: #164e88; font-weight: 600; } +details[open] > summary { margin-bottom: 1rem; } +.evidence { min-width: 16rem; max-width: 50rem; } +dl { margin-block: 1rem; } +dt { font-weight: 650; } +dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } +.matched { border-left: 3px solid #687f93; padding-left: 1rem; margin-block: 1rem; } +@media (max-width: 40rem) { body { padding: .8rem; } .evidence { min-width: 13rem; } } +@media print { body { max-width: none; } .skip, nav { display: none; } .table-scroll { overflow: visible; } } From d163391650324304b83a10dd035fe457f611e482 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:39:31 -0400 Subject: [PATCH 084/126] docs(evaluator): record installed axe-core version --- docs/evaluation-report-verification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 9e86548..20e66fe 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -29,7 +29,7 @@ HTML-like model/reviewer text was copied into a directory containing only | Check | Actual coverage | Result | | --- | --- | --- | | Standalone resources | Chromium 153.0.8010.12, Linux, Playwright offline context | No HTTP requests; report readable | -| Automated accessibility | axe-core 4.11.1, WCAG 2 A/AA and 2.1/2.2 AA tags | No violations or incomplete checks in the tested initial document | +| Automated accessibility | axe-core 4.14.0, WCAG 2 A/AA and 2.1/2.2 AA tags | No violations or incomplete checks in the tested initial document | | Narrow layout | Chromium viewport 320 × 800 CSS pixels | Document and body width 320; findings table scrolls in its own region | | Native disclosure | Chromium, JavaScript disabled, focus first summary and press Enter | Evidence opened; six disclosures were initially collapsed | From b0204c4123052b3c2a0b3da2878f9db939e0d7a4 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:44:56 -0400 Subject: [PATCH 085/126] feat(evaluator): generate standalone Full HTML reports --- README.md | 30 ++- docs/evaluation-report-verification.md | 46 +++++ tests/test_full_execution.py | 2 + tests/test_full_report.py | 86 +++++++- vision_aid/evaluation/cli.py | 2 +- vision_aid/evaluation/full.py | 13 +- vision_aid/evaluation/html_report.py | 70 ++++++- vision_aid/evaluation/review.py | 195 +----------------- vision_aid/evaluation/templates/evidence.html | 10 + vision_aid/evaluation/templates/full.html | 73 +++++++ vision_aid/evaluation/templates/method.html | 7 + vision_aid/evaluation/templates/page.html | 17 +- 12 files changed, 309 insertions(+), 242 deletions(-) create mode 100644 vision_aid/evaluation/templates/evidence.html create mode 100644 vision_aid/evaluation/templates/full.html create mode 100644 vision_aid/evaluation/templates/method.html diff --git a/README.md b/README.md index 6243035..68db031 100644 --- a/README.md +++ b/README.md @@ -634,10 +634,15 @@ uv run visionaid-evaluate report-all \ ``` `report-all` validates every listed execution, child evidence identity, and -page review before writing final combined rates to the parent's `report.md`. -The report lists every original Human finding once, identifies the reviewer, -links page reports, names supporting pages and finding IDs, and totals saved -estimated cost, provider-reported usage, and Unreadable responses. The supplied +page review before writing final combined rates to the parent's `report.html`. +Open that file in a browser. It is a self-contained offline document with every +original Human finding once, the reviewer attribution, and embedded readable +matching evidence. Global findings show a compact table of every page's decision, +including misses and findings Not testable from the page file. Open the independent +page evidence controls to compare the matches, check identities, rationale, and +notes for several pages. The parent needs no child reports or neighboring files. +The command also generates each child's `report.html` and totals saved estimated +cost, provider-reported usage, and Unreadable responses. The supplied audit has 86 originals across 17 pages; repeated Globals produce 198 review rows, while the combined Overall denominator stays 86. @@ -659,9 +664,12 @@ counts in Overall but earns no credit in the expected source's rate. Missing or incomplete execution, invalid reviews, missing evidence, or conflicting testable Global classifications withhold final combined rates. Correct review decisions and run `report-all` again. Regenerating a report -removes the previous parent report before validation, so invalid revised -reviews cannot leave stale combined rates. Available individual reports and -evidence remain. To report one completed child after a later page failure, use: +removes the previous parent HTML and legacy Markdown report before validation, +so invalid revised reviews cannot leave stale combined rates. Available individual +reports and evidence remain. Each HTML file is published only after rendering +succeeds. Successful reporting removes superseded generated Markdown for the +regenerated reports; unrelated historical runs remain untouched. To report one +completed child after a later page failure, use: ```bash uv run visionaid-evaluate report \ @@ -726,8 +734,10 @@ on screen only and absent from va-dat's CSV report. The run root holds the review evidence and scored results. Supporting files are together under `$RUN_DIR/artifacts/`. Without `--live`, the command prepares the prompts but makes no model requests; neither LLM output file exists yet. -The individual `report` command adds `report.html` at the root. The Full -evaluation `report-all` command continues to produce Markdown until its migration. +The `report` command adds `report.html` at an individual run root. The Full +evaluation `report-all` command adds `report.html` at the parent and each child +run root. Reports can regenerate from saved reviewed evidence without model calls; +both commands require explicit reviewer attribution. New runs and reporting use only this layout. Existing run directories remain untouched and require manual reorganization before the updated `report` command @@ -742,7 +752,7 @@ compatibility fallback or automatic migration. | `normalized-llm-findings.json` | LLM findings with `finding_id` values for `review.csv`, read as described above. An Unreadable response can appear in `artifacts/raw-llm-responses.json` without producing a finding here. | | `normalized-programmatic-findings.json` | Checker results converted to findings with `finding_id` values for `review.csv`. | | `run.json` | Configuration, snapshot checksum, prompt list, completion status, format failures, token usage, and estimated cost. `complete: true` does not imply that every response parsed successfully; check `format_failures`. | -| `report.html` | Self-contained individual Evaluation report generated after successful reporting. | +| `report.html` | Self-contained Evaluation report generated after successful reporting; a Full evaluation parent embeds all page matching evidence. | | `artifacts/human-findings.json` | Canonical imported Home and Global Human findings used to validate and score the review. | | `artifacts/pricing.json` | The versioned public model rates and capture date used to estimate this run's cost from reported usage. Older runs retain their original price schedule. | | `artifacts/raw-llm-responses.json` | Live model replies, request outcomes, and reported token usage, including responses that could not be parsed into findings. | diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 20e66fe..060a9a5 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -43,3 +43,49 @@ remain outstanding. Headless keyboard automation and a narrow viewport do not establish those manual results or WCAG conformance. The stakeholder's browser and screen-reader setup is unknown. Combined HTML reports, charts, filtering, search, and complete printing require later ticket verification. + + +## Full HTML report, 2026-10-15 + +Both `report` and `report-all` now publish standalone HTML. The combined report +reuses the validated page reviews and shared evidence templates. No live provider +calls were made during report verification. Existing scoring and review validation +remain authoritative. + +- Full suite: 140 tests passed. `ruff check vision_aid/evaluation tests` and + `git diff --check` passed. +- Public CLI tests preserve the worked eight-original case with rates of 2/4 LLM, + 1/3 Programmatic, and 5/8 Overall. They cover joint matches, cross-catches, + partial evidence across pages, conflicting Global classifications, ownership, + empty pages, zero denominators, reviewer attribution, stale parent removal, + child preservation, and replacement of regenerated Markdown only. +- Parsed HTML assertions cover original source order, one overview row per + original, readable matched IDs, text, check identity and run provenance, + every Global page decision, two disclosure levels, independent panels, and + absence of external resources. Missing required parent provenance fails + rendering without publishing a parent or deleting completed child HTML. +- The saved 17-page run was regenerated in a disposable copy at + `/tmp/va-dat-html-verification/20261011T015629-880f3e7c` with attribution to + its original saved reviewer decisions. It retains 86 originals and 198 page + decisions. Combined counts remain 28/62 LLM, 18/18 Programmatic, and 46/86 + Overall. The parent contains 86 finding disclosures and 119 Global page + disclosures, all initially collapsed. +- `uv build --wheel` packaged the page, Full, shared evidence and method + templates and embedded CSS. A fresh environment at + `/tmp/va-dat-ticket02-installed` generated both scopes from `/tmp`, outside + the checkout, using a separate disposable run copy under + `/tmp/va-dat-ticket02-installed-runs`. + +The parent was copied alone to `/tmp/va-dat-ticket02-standalone/report.html`. +Chromium 153.0.8010.12 opened it via the local-file protocol with networking +disabled and made no HTTP requests. axe-core 4.14.0 reported no violations or +incomplete checks for WCAG 2 A/AA and 2.1/2.2 AA tags in the initial document. +At a 320 × 800 CSS-pixel viewport, document and body widths stayed at 320 pixels; +the table retained its own scroll region. With JavaScript disabled, keyboard +Enter opened a Global finding and two of its page disclosures independently, +leaving both page panels open with readable matching evidence. + +These browser checks are automated. Actual browser zoom, manual keyboard +inspection, and screen-reader verification remain outstanding. Charts, +filtering, search, and complete printing are later ticket work; no WCAG +conformance claim is made. diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py index 7b029f0..86fff0b 100644 --- a/tests/test_full_execution.py +++ b/tests/test_full_execution.py @@ -64,6 +64,7 @@ def approve(prompt): assert preview(full_workspace, "--live", "--model", model) == 0 run, manifest = only_full(full_workspace) saved = children(run, manifest) + assert not (run / "report.html").exists() assert len(approvals) == 1 and manifest["complete"] assert manifest["incomplete_reason"] is None assert len(requests) == sum(len(child["prompts"]) for _, child in saved) > 0 @@ -73,6 +74,7 @@ def approve(prompt): assert child_manifest["complete"] assert child_manifest["configuration"] == manifest["configuration"] assert not (child / "report.md").exists() + assert not (child / "report.html").exists() assert (child / "artifacts/raw-llm-responses.json").exists() for finding in json.loads((child / "normalized-llm-findings.json").read_text()): assert finding["page_url"] == child_manifest["source_url"] diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 8fbc1ba..4f88c8d 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -7,6 +7,7 @@ import openpyxl import pytest +from bs4 import BeautifulSoup from test_evaluation import recording_provider_usage from test_full_execution import children from test_full_preview import URLS, full_workspace, only_full, preview @@ -50,6 +51,7 @@ def refused_report(run, capsys, reviewer="Combined reviewer"): report_all(run, reviewer) assert error.value.code == 2 assert not (run / "report.md").exists() + assert not (run / "report.html").exists() return capsys.readouterr().err @@ -119,7 +121,8 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ """Eight originals score 2/4 LLM, 1/3 Programmatic and 5/8 Overall.""" run, manifest = completed_review assert report_all(run) == 0 - text = (run / "report.md").read_text() + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + text = document.get_text(" ", strip=True) assert "LLM detection rate: Of the 4" in text and "caught 2 (50.0%)" in text assert "Programmatic detection rate: Of the 3" in text and "caught 1 (33.3%)" in text assert "Overall detection rate: Of all 8 Human findings" in text and "caught 5 (62.5%)" in text @@ -129,22 +132,35 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert "published prices captured on 2026-10-09" in text assert "Unreadable responses: 0" in text originals = json.loads((run / "human-findings.json").read_text()) + overview = document.select("#findings > tbody > tr") + assert [row.find("th").get_text(strip=True) for row in overview] == [ + item["problem"] for item in originals["human_findings"]] for finding in originals["human_findings"]: assert text.count(finding["human_finding_id"]) == 1 assert "Not testable on page 1" in text and "Not testable on page 2" in text assert "Partial evidence only on page 1" in text for page in manifest["pages"]: child = run / page["run_dir"] - assert (child / "report.md").is_file() - assert f"]({page['run_dir']}/report.md)" in text + assert (child / "report.html").is_file() assert page["source_url"] in text + tool_findings = { + item["finding_id"]: item + for kind in ("llm", "programmatic") + for item in json.loads((child / f"normalized-{kind}-findings.json").read_text()) + } for row in review_rows(child): for kind in ("llm", "programmatic"): for identity in row[f"{kind}_finding_id"].split(";"): if identity: + finding = tool_findings[identity] assert identity in text + assert finding["problem"] in text + assert finding["prompt"] in text + assert finding["run_id"] in text assert "complete detection on any single page" in text assert "partial matches across pages" in text + assert not document.select("[open]") + assert not document.select("script[src], link[rel=stylesheet], iframe") @pytest.mark.parametrize("problem", [ @@ -217,21 +233,22 @@ def test_empty_page_without_globals_still_has_a_report(full_workspace, monkeypat save_review(child, rows) assert report_all(run) == 0 empty = run / manifest["pages"][2]["run_dir"] - text = (empty / "report.md").read_text() + text = BeautifulSoup((empty / "report.html").read_text(), "html.parser").get_text(" ", strip=True) assert "Overall detection rate: No Human findings were imported for this page" in text - assert "Overall detection rate: Of all 3 Human findings" in (run / "report.md").read_text() + assert "Overall detection rate: Of all 3 Human findings" in BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) def test_invalid_revised_review_withholds_stale_combined_rates(completed_review, capsys): """A report regenerated after an invalid edit must not leave earlier rates.""" run, manifest = completed_review assert report_all(run) == 0 + (run / "report.md").write_text("superseded Markdown") first = run / manifest["pages"][0]["run_dir"] rows = review_rows(first) rows[0]["classification"] = "" save_review(first, rows) assert "invalid classification" in refused_report(run, capsys) - assert (first / "report.md").exists() + assert (first / "report.html").exists() @pytest.mark.parametrize("title", ["Shared defect", "Eligible uncaught"]) @@ -289,7 +306,7 @@ def test_no_eligible_sources_retain_all_originals_in_overall(completed_review): llm_finding_id="", programmatic_finding_id="", match_reason="") save_review(child, rows) assert report_all(run) == 0 - text = (run / "report.md").read_text() + text = BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) assert "LLM detection rate: No Human findings were expected" in text assert "Programmatic detection rate: No Human findings were expected" in text assert "Overall detection rate: Of all 8 Human findings" in text @@ -342,7 +359,7 @@ def create(**options): row.update(classification="llm_eligible", classification_reason="HTML evidence") save_review(child, rows) assert report_all(run) == 0 - assert "Unreadable responses: 6 of 6 LLM checks" in (run / "report.md").read_text() + assert "Unreadable responses: 6 of 6 LLM checks" in BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) def test_report_all_help_explains_review_completion(capsys): @@ -353,3 +370,56 @@ def test_report_all_help_explains_review_completion(capsys): text = " ".join(capsys.readouterr().out.split()) assert "every listed execution" in text and "all page reviews" in text assert "Global" in text and "once" in text + + +def test_global_evidence_includes_every_page_without_deeper_disclosures(completed_review): + """Global evidence exposes misses and not-testable decisions in two levels.""" + run, manifest = completed_review + assert report_all(run) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + rows = document.select("#findings > tbody > tr") + globals_ = [row for row in rows if row.find("th").get_text(strip=True).startswith("Shared")] + assert len(globals_) == 2 + for row in globals_: + outer = row.find("details") + page_panels = outer.find_all("details") + assert len(page_panels) == len(manifest["pages"]) + assert all(not panel.find("details") for panel in page_panels) + assert all(not panel.has_attr("name") for panel in page_panels) + assert all("View page evidence:" in panel.summary.get_text() for panel in page_panels) + compact = outer.find("table") + assert len(compact.select("tbody > tr")) == len(manifest["pages"]) + assert [tr.find("th").get_text(strip=True) for tr in compact.select("tbody > tr")] == [ + page["source_url"] for page in manifest["pages"]] + not_testable = globals_[1].get_text(" ", strip=True) + assert "Not testable from the page file" in not_testable + assert "Not caught" in not_testable + local = next(row for row in rows if row.find("th").get_text(strip=True) == "First local") + assert len(local.find_all("details")) == 1 + + +def test_successful_full_regeneration_replaces_only_its_generated_markdown(completed_review, capsys): + """HTML replaces regenerated Markdown without sweeping historical outputs.""" + run, manifest = completed_review + (run / "report.md").write_text("old combined result") + historical = run.parent / "historical-run" + historical.mkdir() + (historical / "report.md").write_text("historical result") + for page in manifest["pages"]: + (run / page["run_dir"] / "report.md").write_text("old page result") + assert report_all(run) == 0 + assert f"Combined HTML report: {run / 'report.html'}" in capsys.readouterr().out + assert not (run / "report.md").exists() + assert all(not (run / page["run_dir"] / "report.md").exists() for page in manifest["pages"]) + assert (historical / "report.md").read_text() == "historical result" + + +def test_missing_parent_rendering_field_withholds_parent_and_preserves_children(completed_review, capsys): + """Required provenance cannot silently disappear from a published parent.""" + run, manifest = completed_review + assert report_all(run) == 0 + del manifest["human_audit_filename"] + save_json(run / "full.json", manifest) + assert "Cannot render complete report" in refused_report(run, capsys) + assert all((run / page["run_dir"] / "report.html").exists() for page in manifest["pages"]) + assert not list(run.glob(".report.html.*.tmp")) diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py index c27d4c1..2d22456 100644 --- a/vision_aid/evaluation/cli.py +++ b/vision_aid/evaluation/cli.py @@ -305,7 +305,7 @@ def main(argv: Sequence[str] | None = None) -> int: if args.command == "report-all": run = args.run_dir.resolve() report_full_evaluation(run, args.reviewer) - print(f"Combined Markdown report: {run / 'report.md'}") + print(f"Combined HTML report: {run / 'report.html'}") return 0 return _run(args) if args.command == "run" else _report(args) except (OSError, ValueError, KeyError) as error: diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py index c372be3..fcae57d 100644 --- a/vision_aid/evaluation/full.py +++ b/vision_aid/evaluation/full.py @@ -11,6 +11,7 @@ import requests +from vision_aid.evaluation.html_report import write_full_report, write_page_report from vision_aid.evaluation.human_audit import import_all_human_findings, list_audit_urls from vision_aid.evaluation.llm_audit import ( USAGE_KEYS, @@ -20,7 +21,6 @@ write_json, ) from vision_aid.evaluation.page import PageReview, prepare_page, validated_page_review -from vision_aid.evaluation.review import write_combined_report, write_report from vision_aid.evaluation.schemas import HumanAudit @@ -333,15 +333,14 @@ def report_full_evaluation(run: Path, reviewer: str) -> None: raise ValueError("A reviewer description is required") run = run.resolve() # Regeneration cannot leave a previous final report beside invalid decisions. + (run / "report.html").unlink(missing_ok=True) (run / "report.md").unlink(missing_ok=True) manifest = json.loads((run / "full.json").read_text()) original = json.loads((run / "human-findings.json").read_text()) reviews = _validated_full_reviews(run, manifest, original) rows = _pooled_rows(original["human_findings"], manifest["pages"], reviews) for page, review in zip(manifest["pages"], reviews, strict=True): - write_report( - run / page["run_dir"] / "report.md", review.manifest, - review.rows, review.llm, review.programmatic, reviewer, - review.schedule.price_description, - ) - write_combined_report(run / "report.md", manifest, rows, reviews, reviewer) + child = run / page["run_dir"] + write_page_report(child / "report.html", review, reviewer) + (child / "report.md").unlink(missing_ok=True) + write_full_report(run / "report.html", manifest, rows, reviews, reviewer) diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py index 6d2baa0..01fc656 100644 --- a/vision_aid/evaluation/html_report.py +++ b/vision_aid/evaluation/html_report.py @@ -110,7 +110,65 @@ def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: } +def full_data( + manifest: dict[str, Any], pooled: list[dict[str, Any]], + reviews: list[PageReview], reviewer: str, +) -> dict[str, Any]: + """Combine validated presentation rows without changing pooled scoring.""" + pages = [page_data(review, reviewer) for review in reviews] + grouped: dict[str, list[dict[str, Any]]] = {} + for page in pages: + for row in page["rows"]: + grouped.setdefault(row["human"]["human_finding_id"], []).append({ + **row, "source_url": page["manifest"]["source_url"], + "run_id": page["manifest"]["run_id"], + }) + counts = detection_counts(pooled) + rows = [] + for row in pooled: + rows.append({ + "human": row["human_finding"], + "expected": EXPECTED[row["decision"]["classification"]], + "llm_outcome": "Caught" if row["matches"]["llm"] else "Not caught", + "programmatic_outcome": "Caught" if row["matches"]["programmatic"] else "Not caught", + "pages": grouped[row["human_finding"]["human_finding_id"]], + }) + return { + "title": "Full accessibility evaluation", "full": True, + "manifest": manifest, "reviewer": reviewer, "rows": rows, "pages": pages, + "rates": [{ + "label": label.removesuffix(" checks"), **counts[kind], + "description": _rate(label, counts[kind]["caught"], counts[kind]["eligible"]).replace( + "the page's HTML", "the evaluated pages' HTML"), + } for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks"), ("overall", "Overall"))], + "cost": f"{Decimal(manifest['estimated_cost_usd']):.2f}", + "pricing_description": ", ".join(dict.fromkeys(page["pricing_description"] for page in pages)), + "pricing_schedules": list({ + page["pricing_identity"]: { + "version": page["pricing_version"], "identity": page["pricing_identity"], + "sources": page["pricing_sources"], + } for page in pages + }.values()), + "usage_description": _usage_description(manifest["usage"]), + "unreadable": sum(page["unreadable"] for page in pages), + "prompts": sum(page["prompts"] for page in pages), + } + + +def write_full_report( + path: Path, manifest: dict[str, Any], pooled: list[dict[str, Any]], + reviews: list[PageReview], reviewer: str, +) -> None: + """Render every accepted page decision into the standalone parent.""" + _write_html(path, "full.html", full_data(manifest, pooled, reviews, reviewer)) + + def write_page_report(path: Path, review: PageReview, reviewer: str) -> None: + """Publish a standalone individual Evaluation report.""" + _write_html(path, "page.html", page_data(review, reviewer)) + + +def _write_html(path: Path, template: str, data: dict[str, Any]) -> None: """Render completely, then atomically replace the report in its directory.""" environment = Environment( loader=PackageLoader("vision_aid.evaluation", "templates"), @@ -118,20 +176,14 @@ def write_page_report(path: Path, review: PageReview, reviewer: str) -> None: undefined=StrictUndefined, ) try: - content = environment.get_template("page.html").render( - **page_data(review, reviewer) - ) + content = environment.get_template(template).render(**data) except TemplateError as error: raise ValueError(f"Cannot render complete report: {error}") from error temporary = None try: with tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - suffix=".tmp", - delete=False, + mode="w", encoding="utf-8", dir=path.parent, + prefix=f".{path.name}.", suffix=".tmp", delete=False, ) as stream: temporary = Path(stream.name) stream.write(content) diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py index 48e6f15..6ab6e33 100644 --- a/vision_aid/evaluation/review.py +++ b/vision_aid/evaluation/review.py @@ -1,12 +1,10 @@ -"""One editable CSV for human decisions and one deterministic Markdown report.""" +"""Validate editable Human review decisions and calculate deterministic counts.""" from __future__ import annotations import csv -from decimal import Decimal from pathlib import Path from typing import Any -from urllib.parse import urlsplit from vision_aid.evaluation.human_audit import ELIGIBILITY_VALUES, HEADERS @@ -159,14 +157,6 @@ def detection_counts(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]: return counts -def _md(value: object) -> str: - """Escape inline text for Markdown, including raw HTML, keeping line breaks.""" - text = str(value) - for character in "\\`*_[]|<>": - text = text.replace(character, "\\" + character) - return text.replace("\n", "
") - - def _usage_description(usage: dict[str, int]) -> str: """Describe reported input, output, and nonzero cache token totals.""" tokens = [ @@ -180,186 +170,3 @@ def _usage_description(usage: dict[str, int]) -> str: if usage[key]: tokens.append(f"{usage[key]} {label} tokens") return ", ".join(tokens) - - -def write_report( - path: Path, - manifest: dict[str, Any], - rows: list[dict[str, Any]], - llm: list[dict[str, Any]], - programmatic: list[dict[str, Any]], - reviewer: str, - pricing_description: str, -) -> None: - """Explain the review's catches, misses, and scoring in source order.""" - if not manifest["complete"]: - raise ValueError("Incomplete Evaluation runs cannot produce a final report") - findings = {item["finding_id"]: item for item in llm + programmatic} - expected = { - "llm_eligible": "LLM", - "programmatic": "Programmatic checks", - "unavailable_evidence": "Not testable", - "ambiguous": "Not testable", - } - covered = [row for row in rows if any(row["matches"].values())] - counts = detection_counts(rows) - page_scope = ( - "This evaluation covers the homepage only" - if urlsplit(manifest["source_url"]).path in {"", "/"} - else f"This evaluation covers the saved page at {_md(manifest['source_url'])}" - ) - lines = [ - f"# LLM accessibility evaluation: {_md(manifest['source_url'])}", - "", - "## How to read this report", - "", - "The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.", - "This API uses LLM checks and Programmatic checks. LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.", - "LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.", - f"The tool saw only saved HTML, not a live browser or screen reader. {page_scope}, including Global findings that apply there.", - "A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.", - "", - "## Summary", - "", - f"Model: {_md(manifest['configuration']['model'])}. Run ID: {_md(manifest['run_id'])}.", - f"Review decisions by: {_md(reviewer)}", - "", - ] - for kind, part in ( - ("llm", "LLM"), - ("programmatic", "Programmatic checks"), - ): - lines.extend([ - _rate(part, counts[kind]["caught"], counts[kind]["eligible"]), "" - ]) - lines.extend([ - _rate("Overall", counts["overall"]["caught"], counts["overall"]["eligible"]), - "", - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated using provider-reported token usage and the {_md(pricing_description)}. Reported totals: {_usage_description(manifest['usage'])}. The actual bill may differ slightly.", - "", - ]) - if manifest["format_failures"]: - checks = ", ".join(_md(name.replace("_", " ") + " check") for name in manifest["format_failures"]) - lines.extend([f"{len(manifest['format_failures'])} of {len(manifest['prompts'])} LLM checks returned an Unreadable response that the tool would skip: {checks}.", ""]) - lines.extend([ - "## Human findings", "", - "| Issue Title | WCAG | Expected to be caught by | LLM | Programmatic checks |", - "| --- | --- | --- | --- | --- |", - ]) - for row in rows: - human = row["human_finding"] - marks = ["✓" if row["matches"][kind] else "✗" for kind in ("llm", "programmatic")] - lines.append(f"| {_md(human['problem'])} | {_md(human['raw_evidence']['WCAG Sc'])} | {expected[row['decision']['classification'].strip()]} | {' | '.join(marks)} |") - lines.extend([ - "", - "The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part still gets a ✓ and counts toward the Overall detection rate, which counts each Human finding once out of all imported Human findings.", - ]) - groups = { - "What each part of the tool caught": covered, - "Missed": [ - row for row in rows - if expected[row["decision"]["classification"].strip()] != "Not testable" - and not any(row["matches"].values()) - ], - "Not testable from the page file": [ - row for row in rows - if expected[row["decision"]["classification"].strip()] == "Not testable" - ], - } - for heading, group in groups.items(): - lines.extend(["", f"## {heading}", ""]) - if heading == "Not testable from the page file": - lines.extend(["These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator.", ""]) - for row in group: - decision = row["decision"] - matches = row["matches"] - lines.extend([f"### {_md(row['human_finding']['problem'])}", ""]) - for kind in ("llm", "programmatic"): - for identity in matches[kind]: - finding = findings[identity] - check = finding["prompt"].replace("_", " ") + " check" - if finding.get("screen_only"): - check += "; shown on screen only, not in the tool's CSV report" - label = ( - f"LLM ({check})" - if kind == "llm" else f"Programmatic check {finding['prompt']}" - ) - lines.extend([f"- {_md(label)}: {_md(finding['problem'])}", "", f" - Why this counts: {_md(decision['match_reason'])}"]) - if heading == "Not testable from the page file": - lines.append(_md(decision["classification_reason"])) - elif not any(matches.values()): - for field, label in ( - ("classification_reason", "Why it was expected to be caught"), - ("review_notes", "Why it was missed"), - ): - if decision[field].strip(): - lines.append(f"- {label}: {_md(decision[field])}") - lines.append("") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def write_combined_report( - path: Path, - manifest: dict[str, Any], - rows: list[dict[str, Any]], - page_reviews: list[Any], - reviewer: str, -) -> None: - """Describe pooled original findings with every supporting page decision.""" - counts = detection_counts(rows) - prices = ", ".join(dict.fromkeys(review.schedule.price_description for review in page_reviews)) - failures = sum(len(review.manifest["format_failures"]) for review in page_reviews) - prompts = sum(len(review.manifest["prompts"]) for review in page_reviews) - lines = [ - "# Full accessibility evaluation", "", - f"Model: {_md(manifest['configuration']['model'])}. Run ID: {_md(manifest['run_id'])}.", - f"Human audit: {_md(manifest['human_audit_filename'])}; SHA-256: {_md(manifest['human_audit_sha256'])}.", - f"Review decisions by: {_md(reviewer)}", "", - "The tool evaluated only saved HTML for every page listed in the Human audit, with one fixed model. LLM checks use model judgment; Programmatic checks apply fixed rules to HTML.", - "Each original Human finding counts once. Non-Global findings use their assigned page. Global findings earn credit through complete detection on any single page, independently for each detection source. Several catches still count once; partial matches across pages are never combined.", - "Eligibility is established independently of detection. If any page is testable for a Global, the finding enters that expected source's denominator once; all testable pages must agree on the source. Findings not testable on any page remain in Overall but are excluded from LLM and Programmatic rates. Cross-catches count toward Overall, which counts either source's catch once.", - "", "## Summary", "", - ] - for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks"), ("overall", "Overall")): - lines.extend([ - _rate(label, counts[kind]["caught"], counts[kind]["eligible"]).replace( - "the page's HTML", "the evaluated pages' HTML" - ), "", - ]) - lines.extend([ - f"Estimated run cost: ${Decimal(manifest['estimated_cost_usd']):.2f}, calculated from provider-reported usage using {_md(prices)}. Reported totals: {_usage_description(manifest['usage'])}. The actual bill may differ.", - f"Unreadable responses: {failures} of {prompts} LLM checks. These responses yield no LLM findings and do not change execution completion.", - "", "## Evaluated pages", "", - ]) - for page in manifest["pages"]: - lines.append(f"- [{_md(page['source_url'])}]({page['run_dir']}/report.md)") - lines.extend(["", "## Original Human findings", ""]) - expected = { - "llm_eligible": "LLM", "programmatic": "Programmatic checks", - "unavailable_evidence": "Not testable from the page file", - "ambiguous": "Not testable from the page file", - } - for row in rows: - finding = row["human_finding"] - classification = row["decision"]["classification"] - marks = {kind: "✓" if row["matches"][kind] else "✗" for kind in ("llm", "programmatic")} - lines.extend([ - f"### {_md(finding['problem'])}", "", - f"Human finding ID: {_md(finding['human_finding_id'])}. Source: {_md(finding['source_sheet'])}, row {finding['source_row']}; scope: {_md(finding['page_scope'])}; WCAG: {_md(finding['raw_evidence']['WCAG Sc'])}.", - f"Expected to be caught by: {expected[classification]}. LLM: {marks['llm']}; Programmatic checks: {marks['programmatic']}.", "", - ]) - for page_row in row["page_reviews"]: - decision = page_row["decision"] - source_url = _md(page_row["source_url"]) - lines.append( - f"- Page {source_url}: {expected[decision['classification'].strip()]}" - + (f". {_md(decision['classification_reason'])}" if decision['classification_reason'].strip() else "") - ) - for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks")): - identities = page_row["matches"][kind] - if identities: - lines.append(f" - {label} full match: {', '.join(_md(identity) for identity in identities)}. Why this counts: {_md(decision['match_reason'])}") - if decision["review_notes"].strip(): - lines.append(f" - Review notes: {_md(decision['review_notes'])}") - lines.append("") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/vision_aid/evaluation/templates/evidence.html b/vision_aid/evaluation/templates/evidence.html new file mode 100644 index 0000000..5cb17f7 --- /dev/null +++ b/vision_aid/evaluation/templates/evidence.html @@ -0,0 +1,10 @@ +{% macro human_evidence(row) %} +

Original Human finding

+
Human finding ID
{{ row.human.human_finding_id }}
Source
{{ row.human.source_sheet }}, row {{ row.human.source_row }}; scope: {{ row.human.page_scope }}; element: {{ row.human.source_element }}
+{% for label, value in row.human.raw_evidence.items() %}
{{ label }}
{{ value }}
{% endfor %}
+{% endmacro %} +{% macro decision_evidence(row) %} +

Review decision

+
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
Review notes
{{ row.decision.review_notes }}
+{% for finding in row.matched %}

{{ finding.source }} matching evidence

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
Page provenance
{{ finding.page_url }}; run {{ finding.run_id }}
Element
{{ finding.element }}
Location
{{ finding.location }}
{% endfor %} +{% endmacro %} diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html new file mode 100644 index 0000000..8e0364f --- /dev/null +++ b/vision_aid/evaluation/templates/full.html @@ -0,0 +1,73 @@ +{% from 'evidence.html' import human_evidence, decision_evidence %} + + + + + +{{ title }}: {{ manifest.run_id }} + + + + +
+

{{ title }}

+

This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.

+

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

+

Review decisions by: {{ reviewer }}

+ +
+
+
+

Detection results

+{% for rate in rates %}

{{ rate.description }}

{% endfor %} +

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each original Human finding once.

+

Each original Human finding counts once. Page-local findings use their assigned page. Global findings earn credit through complete detection on any single page, independently for each detection source. Several catches still count once; partial matches across pages are never combined.

+
+
+

Human findings

+

Each original Human finding appears once in source order. Evidence starts collapsed. Open View evidence for the original Human finding and its review decisions. Global findings include every page decision, with independently expandable page evidence.

+
+ + + + +{% for row in rows %} + + + +{% endfor %} +
Original Human findings and combined detection outcomes
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksEvidence
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
+{{ human_evidence(row) }} +{% if row.human.page_scope == 'global' %} +

Combined result

+

Expected source: {{ row.expected }}. Combined LLM: {{ row.llm_outcome }}. Combined Programmatic checks: {{ row.programmatic_outcome }}. Any complete catch on one page earns combined credit once for that source; partial evidence across pages earns none.

+
+ +{% for page in row.pages %}{% endfor %} +
Every page decision for {{ row.human.problem }}
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
+{% for page in row.pages %}
View page evidence: {{ page.source_url }} for {{ row.human.problem }} +

Page: {{ page.source_url }}; run {{ page.run_id }}. LLM: {{ page.llm_outcome }}. Programmatic checks: {{ page.programmatic_outcome }}.

+{{ decision_evidence(page) }} +
{% endfor %} +{% else %} +{% for page in row.pages %}

Assigned page: {{ page.source_url }}; run {{ page.run_id }}.

{{ decision_evidence(page) }}{% endfor %} +{% endif %} +
+
+{% include 'method.html' %} +

Full evaluation scoring

+

Eligibility is established independently of detection. If any page is testable for a Global finding, the finding enters that expected source's denominator once. All testable pages must agree on the source. Findings not testable on any page remain in Overall and outside the LLM and Programmatic denominators.

+

Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged.

+
+

Cost and provenance

+

Estimated run cost: ${{ cost }}, calculated using authoritative saved page totals, provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

+

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

+
Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
+{% for schedule in pricing_schedules %}
Saved pricing version
{{ schedule.version }}
Saved pricing SHA-256
{{ schedule.identity }}
{% for source in schedule.sources %}
Pricing source address
{{ source }}
{% endfor %}{% endfor %} +
+

Evaluated pages in source order

+
{% for page in pages %}
{{ page.manifest.source_url }}
Run: {{ page.manifest.run_id }}; Benchmark snapshot SHA-256: {{ page.manifest.snapshot_sha256 }}. Unreadable responses: {{ page.unreadable }} of {{ page.prompts }} LLM checks{% if page.unreadable %}; {{ page.failed_checks }}{% endif %}.
{% endfor %}
+
+
+ + diff --git a/vision_aid/evaluation/templates/method.html b/vision_aid/evaluation/templates/method.html new file mode 100644 index 0000000..41325fa --- /dev/null +++ b/vision_aid/evaluation/templates/method.html @@ -0,0 +1,7 @@ +

How to read this report

+

The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.

+

LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.

+

LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.

+

The tool saw only saved HTML, not a live browser or screen reader. A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.

+

Not testable from the page file: These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator. Their recorded explanations appear in the evidence.

+
diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html index ff37e1a..25184c4 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html @@ -1,3 +1,4 @@ +{% from 'evidence.html' import human_evidence, decision_evidence %} @@ -34,25 +35,15 @@

Human findings

{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
-

Original Human finding

-
Human finding ID
{{ row.human.human_finding_id }}
Source
{{ row.human.source_sheet }}, row {{ row.human.source_row }}; scope: {{ row.human.page_scope }}; element: {{ row.human.source_element }}
-{% for label, value in row.human.raw_evidence.items() %}
{{ label }}
{{ value }}
{% endfor %}
-

Review decision

-
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
Review notes
{{ row.decision.review_notes }}
-{% for finding in row.matched %}

{{ finding.source }} matching evidence

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
Page provenance
{{ finding.page_url }}; run {{ finding.run_id }}
Element
{{ finding.element }}
Location
{{ finding.location }}
{% endfor %} +{{ human_evidence(row) }} +{{ decision_evidence(row) }}
{% endfor %} {% if not rows %}

No Human findings were imported for this page.

{% endif %} -

How to read this report

-

The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.

-

LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.

-

LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.

-

The tool saw only saved HTML, not a live browser or screen reader. A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.

-

Not testable from the page file: These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator. Their recorded explanations appear in the evidence.

-
+{% include 'method.html' %}

Cost and provenance

Estimated run cost: ${{ cost }}, calculated using provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

From 43a672ac8bf5a4e27618f5bb9b16be73b76b9e94 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:45:06 -0400 Subject: [PATCH 086/126] docs(evaluator): record complete Full report evidence check --- docs/evaluation-report-verification.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 060a9a5..94ce014 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -68,8 +68,10 @@ remain authoritative. `/tmp/va-dat-html-verification/20261011T015629-880f3e7c` with attribution to its original saved reviewer decisions. It retains 86 originals and 198 page decisions. Combined counts remain 28/62 LLM, 18/18 Programmatic, and 46/86 - Overall. The parent contains 86 finding disclosures and 119 Global page - disclosures, all initially collapsed. + Overall. Independent inspection resolved all 187 matched ID occurrences to + readable problem text and confirmed all 86 original IDs and titles. The parent + contains 86 finding disclosures and 119 Global page disclosures, all initially + collapsed. - `uv build --wheel` packaged the page, Full, shared evidence and method templates and embedded CSS. A fresh environment at `/tmp/va-dat-ticket02-installed` generated both scopes from `/tmp`, outside From 5305e2cebcf3b8ccd58465f3f2be9852b3bb1b81 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:50:38 -0400 Subject: [PATCH 087/126] feat(evaluator): visualize detection rates in offline reports --- README.md | 21 ++++++++++ docs/evaluation-report-verification.md | 44 ++++++++++++++++++++ tests/test_evaluation.py | 18 ++++++++ tests/test_full_report.py | 27 ++++++++++++ vision_aid/evaluation/html_report.py | 46 ++++++++++++--------- vision_aid/evaluation/templates/charts.css | 10 +++++ vision_aid/evaluation/templates/charts.html | 33 +++++++++++++++ vision_aid/evaluation/templates/full.html | 5 ++- vision_aid/evaluation/templates/page.html | 4 +- 9 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 vision_aid/evaluation/templates/charts.css create mode 100644 vision_aid/evaluation/templates/charts.html diff --git a/README.md b/README.md index 68db031..481caba 100644 --- a/README.md +++ b/README.md @@ -677,6 +677,27 @@ uv run visionaid-evaluate report \ --reviewer "Name and how this page was reviewed" ``` +### Reading the detection charts + +Both report scopes lead with an LLM detection bar, followed by Programmatic and +Overall results. Bar length uses a common 0% to 100% scale. Every bar shows its +caught count, eligible count and percentage, with the same values in a labelled +semantic table. A rate with no eligible findings has an explanation and no bar. +The Overall denominator includes all imported Human findings, including findings +Not testable from the page file. Source-specific rates count only catches by that +source among the findings expected from it; cross-catches still earn Overall +credit. + +Full reports also compare page LLM results in Human audit page order. Page +eligibility varies, and Global findings repeat in page results while counting +once in the combined result. Do not add page counts or average page percentages +to calculate the combined result. The comparison is not a controlled ranking of +pages or model quality. Individual reports omit this comparison. + +Charts and their exact-data tables work offline without JavaScript. Labels and +values convey the results without relying on color; tables have their own +horizontal scroll regions on narrow screens. + ### How the evaluator reads LLM responses The evaluator reads each LLM response using va-dat's own parsing and reporting diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 94ce014..abe3131 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -91,3 +91,47 @@ These browser checks are automated. Actual browser zoom, manual keyboard inspection, and screen-reader verification remain outstanding. Charts, filtering, search, and complete printing are later ticket work; no WCAG conformance claim is made. + +## Detection charts, 2026-10-15 + +Both commands now render labelled detection overview bars with exact counts, +percentages and semantic tables. The LLM result appears first with a stronger +border and larger label. Programmatic and Overall denominators remain separate. +Python prepares all chart percentages from the existing deterministic counts; +charts contain no script or external dependency. A zero denominator has a text +explanation and no bar. Full reports include page LLM results in original page +order, with an explanation of varying eligibility, repeated Globals and the +limits of comparing pages. + +- Full suite: 140 tests passed. Existing public CLI fixtures now assert chart + and table results for individual cross-catches, the worked combined 2/4 LLM, + 1/3 Programmatic and 5/8 Overall case, and page LLM counts of 1/3, 1/3 and 1/2. + Empty original sets and empty page results retain undefined source rates in + words. Nonempty Overall denominators still report a truthful 0.0% when no + findings were caught. Individual reports omit the page comparison. +- `ruff check vision_aid/evaluation tests` and `git diff --check`: passed. +- `uv build --wheel` bundled `charts.html` and `charts.css` with the shared + templates. The wheel was installed in a fresh environment at + `/tmp/va-dat-ticket03-installed`; its `report-all` and `report` commands + generated both scopes from `/tmp`, outside the checkout, against a separate + disposable copy at `/tmp/va-dat-ticket03-installed-runs`. No saved review + decisions were changed and no live model calls were made. +- The installed parent retained 28/62 LLM, 18/18 Programmatic and 46/86 Overall; + the first individual page retained 3/9 LLM, 5/5 Programmatic and 8/16 Overall. + Copied standalone files contain only their respective `report.html` under + `/tmp/va-dat-ticket03-standalone/full` and `/tmp/va-dat-ticket03-standalone/page`. + +Chromium 153.0.8010.12 opened both copied files through `file:` in offline +contexts. axe-core 4.14.0 reported no violations or incomplete checks using +WCAG 2 A/AA and 2.1/2.2 AA tags. No HTTP requests occurred. Browser assertions +located the named figures and captioned tables, measured rendered bar lengths +against the displayed 45.2%, 100.0%, 53.5% parent values and 33.3%, 100.0%, 50.0% +page values, and confirmed 17 page-comparison table rows. With JavaScript +disabled these labels, values, tables and graphics remained available. A +320 × 800 CSS-pixel viewport retained a 320-pixel document width in both scopes; +tables retained their own scroll regions. Automated keyboard Enter opened the +first native disclosure in both scopes. + +These checks do not replace actual screen-reader reading, manual keyboard +inspection or actual browser zoom. Those checks remain outstanding under the +assistive-technology verification ticket, and no WCAG conformance claim is made. diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 4cb729c..884e91c 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -661,6 +661,17 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): assert "they caught 1 (50.0%)" in text assert "Of all 6 Human findings, either part of the tool caught 3 (50.0%)" in text document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + chart = document.find("figure", attrs={"aria-labelledby": "detection-chart-title"}) + assert chart is not None + assert "LLM: 1 caught / 2 eligible; 50.0%" in chart.get_text(" ", strip=True) + table = document.find("caption", string="Exact detection overview data").parent + assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] + for row in table.tbody.find_all("tr")] == [ + ["LLM", "1", "2", "50.0%", "Human findings expected to be detected by the LLM"], + ["Programmatic", "1", "2", "50.0%", "Human findings expected to be detected by Programmatic checks"], + ["Overall", "3", "6", "50.0%", "All imported Human findings, including findings Not testable from the page file"], + ] + assert document.find("caption", string="Exact page LLM detection data") is None rows = document.select("#findings tbody > tr") assert [cell.get_text(" ", strip=True) for cell in rows[2].find_all(["th", "td"], recursive=False)[:5]] == [ "Skipped heading", "1.3.1", "Programmatic checks", "Caught", "Not caught", @@ -904,4 +915,11 @@ def test_report_with_no_human_findings_has_no_false_zero_rates(workspace, monkey assert "No Human findings were expected to be detected by the Programmatic checks" in text assert "(0.0%)" not in text document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + table = document.find("caption", string="Exact detection overview data").parent + assert [row.find_all("td")[2].get_text(strip=True) for row in table.tbody.find_all("tr")] == [ + "Not defined: no imported Human findings" if index == 2 else "Not defined: no eligible Human findings" + for index in range(3) + ] + chart = document.find("figure", attrs={"aria-labelledby": "detection-chart-title"}) + assert "0.0%" not in chart.get_text(" ", strip=True) assert not document.select("#findings tbody tr") diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 4f88c8d..26967b8 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -126,6 +126,24 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert "LLM detection rate: Of the 4" in text and "caught 2 (50.0%)" in text assert "Programmatic detection rate: Of the 3" in text and "caught 1 (33.3%)" in text assert "Overall detection rate: Of all 8 Human findings" in text and "caught 5 (62.5%)" in text + overview = document.find("caption", string="Exact detection overview data").parent + assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])[:4]] + for row in overview.tbody.find_all("tr")] == [ + ["LLM", "2", "4", "50.0%"], + ["Programmatic", "1", "3", "33.3%"], + ["Overall", "5", "8", "62.5%"], + ] + comparison = document.find("caption", string="Exact page LLM detection data") + assert comparison is not None + assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] + for row in comparison.parent.tbody.find_all("tr")] == [ + [URLS[0], "1", "3", "33.3%"], + [URLS[1], "1", "3", "33.3%"], + [URLS[2], "1", "2", "50.0%"], + ] + assert "Repeated Global findings contribute to each page's results" in text + assert "page rates must not be added or averaged" in text + assert "not a controlled ranking of pages or model quality" in text assert "Review decisions by: Combined reviewer" in text assert "Estimated run cost: $0.00" in text assert "1000 input tokens" in text and "100 output tokens" in text @@ -236,6 +254,15 @@ def test_empty_page_without_globals_still_has_a_report(full_workspace, monkeypat text = BeautifulSoup((empty / "report.html").read_text(), "html.parser").get_text(" ", strip=True) assert "Overall detection rate: No Human findings were imported for this page" in text assert "Overall detection rate: Of all 3 Human findings" in BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) + parent = BeautifulSoup((run / "report.html").read_text(), "html.parser") + comparison = parent.find("caption", string="Exact page LLM detection data").parent + assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] + for row in comparison.tbody.find_all("tr")] == [ + [URLS[0], "0", "0", "Not defined: no eligible Human findings"], + [URLS[1], "0", "0", "Not defined: no eligible Human findings"], + [URLS[2], "0", "0", "Not defined: no eligible Human findings"], + ] + def test_invalid_revised_review_withholds_stale_combined_rates(completed_review, capsys): diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py index 01fc656..0ed8bfc 100644 --- a/vision_aid/evaluation/html_report.py +++ b/vision_aid/evaluation/html_report.py @@ -29,6 +29,31 @@ } +def _chart_rates(counts: dict[str, dict[str, int]], *, full: bool = False) -> list[dict[str, Any]]: + """Prepare chart labels and widths from the existing deterministic counts.""" + rates = [] + for kind, label, denominator in ( + ("llm", "LLM", "Human findings expected to be detected by the LLM"), + ("programmatic", "Programmatic checks", "Human findings expected to be detected by Programmatic checks"), + ("overall", "Overall", "All imported Human findings, including findings Not testable from the page file"), + ): + caught, eligible = counts[kind]["caught"], counts[kind]["eligible"] + percentage = f"{caught / eligible * 100:.1f}" if eligible else None + description = _rate(label, caught, eligible) + if full: + description = description.replace("the page's HTML", "the evaluated pages' HTML") + rates.append({ + "label": label.removesuffix(" checks"), **counts[kind], + "description": description, "denominator": denominator, + "bar_percent": percentage, + "percentage": percentage + "%" if percentage is not None else ( + "Not defined: no imported Human findings" if kind == "overall" + else "Not defined: no eligible Human findings" + ), + }) + return rates + + def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: """Resolve validated evidence and deterministic scores before rendering.""" manifest = review.manifest @@ -71,20 +96,7 @@ def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: "matched": matched, } ) - rates = [ - { - "label": label.removesuffix(" checks"), - **counts[kind], - "description": _rate( - label, counts[kind]["caught"], counts[kind]["eligible"] - ), - } - for kind, label in ( - ("llm", "LLM"), - ("programmatic", "Programmatic checks"), - ("overall", "Overall"), - ) - ] + rates = _chart_rates(counts) return { "title": "LLM accessibility evaluation", "manifest": manifest, @@ -136,11 +148,7 @@ def full_data( return { "title": "Full accessibility evaluation", "full": True, "manifest": manifest, "reviewer": reviewer, "rows": rows, "pages": pages, - "rates": [{ - "label": label.removesuffix(" checks"), **counts[kind], - "description": _rate(label, counts[kind]["caught"], counts[kind]["eligible"]).replace( - "the page's HTML", "the evaluated pages' HTML"), - } for kind, label in (("llm", "LLM"), ("programmatic", "Programmatic checks"), ("overall", "Overall"))], + "rates": _chart_rates(counts, full=True), "cost": f"{Decimal(manifest['estimated_cost_usd']):.2f}", "pricing_description": ", ".join(dict.fromkeys(page["pricing_description"] for page in pages)), "pricing_schedules": list({ diff --git a/vision_aid/evaluation/templates/charts.css b/vision_aid/evaluation/templates/charts.css new file mode 100644 index 0000000..f4107f2 --- /dev/null +++ b/vision_aid/evaluation/templates/charts.css @@ -0,0 +1,10 @@ +.detection-chart { margin: 1rem 0; } +.chart-bars { list-style: none; padding: 0; } +.chart-bars li { margin-block: 1rem; padding: .75rem; border: 1px solid #687f93; overflow-wrap: anywhere; } +.chart-bars .chart-primary { border: 3px solid #164e88; } +.chart-primary .chart-value { font-size: 1.2rem; } +.chart-value { margin: 0 0 .5rem; } +.chart-track { width: 100%; height: 1.4rem; background: #edf2f6; border: 1px solid #687f93; } +.chart-fill { display: block; height: 100%; background: #164e88; } +@media (forced-colors: active) { .chart-fill { background: Highlight; } } +@media print { .chart-track, .chart-fill { print-color-adjust: exact; } .chart-bars li { break-inside: avoid; } } diff --git a/vision_aid/evaluation/templates/charts.html b/vision_aid/evaluation/templates/charts.html new file mode 100644 index 0000000..dffc400 --- /dev/null +++ b/vision_aid/evaluation/templates/charts.html @@ -0,0 +1,33 @@ +{% macro rate_bar(label, rate, primary=false) %} + +

{{ label }}: {{ rate.caught }} caught / {{ rate.eligible }} eligible; {{ rate.percentage }}

+{% if rate.bar_percent is not none %}{% endif %} + +{% endmacro %} + +{% macro detection_overview(rates) %} +
+

Detection overview

+

Bar length shows the detection rate on a scale from 0% to 100%. The LLM result leads; each source uses its own denominator. No bar is drawn when the rate is not defined.

+
    {% for rate in rates %}{{ rate_bar(rate.label, rate, loop.first) }}{% endfor %}
+
+ + +{% for rate in rates %}{% endfor %} +
Exact detection overview data
Detection sourceCaughtEligibleDetection rateDenominator
{{ rate.label }}{{ rate.caught }}{{ rate.eligible }}{{ rate.percentage }}{{ rate.denominator }}
+
+{% endmacro %} + +{% macro page_comparison(pages) %} +
+

LLM detection by page

+

Pages appear in Human audit order. Each bar uses a 0% to 100% scale and the page's LLM-eligible findings. Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged. This comparison is not a controlled ranking of pages or model quality.

+

No bar is drawn where no Human findings were eligible for the LLM. Exact values follow in the table.

+
    {% for page in pages %}{{ rate_bar(page.manifest.source_url, page.rates[0]) }}{% endfor %}
+
+ + +{% for page in pages %}{% endfor %} +
Exact page LLM detection data
PageLLM caughtLLM eligibleLLM detection rate
{{ page.manifest.source_url }}{{ page.rates[0].caught }}{{ page.rates[0].eligible }}{{ page.rates[0].percentage }}
+
+{% endmacro %} diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html index 8e0364f..854f5ab 100644 --- a/vision_aid/evaluation/templates/full.html +++ b/vision_aid/evaluation/templates/full.html @@ -1,11 +1,12 @@ {% from 'evidence.html' import human_evidence, decision_evidence %} +{% from 'charts.html' import detection_overview, page_comparison %} {{ title }}: {{ manifest.run_id }} - + @@ -19,9 +20,11 @@

{{ title }}

Detection results

+{{ detection_overview(rates) }} {% for rate in rates %}

{{ rate.description }}

{% endfor %}

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each original Human finding once.

Each original Human finding counts once. Page-local findings use their assigned page. Global findings earn credit through complete detection on any single page, independently for each detection source. Several catches still count once; partial matches across pages are never combined.

+{{ page_comparison(pages) }}

Human findings

diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html index 25184c4..0c8bdf1 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html @@ -1,11 +1,12 @@ {% from 'evidence.html' import human_evidence, decision_evidence %} +{% from 'charts.html' import detection_overview, page_comparison %} {{ title }}: {{ manifest.source_url }} - + @@ -19,6 +20,7 @@

{{ title }}

Detection results

+{{ detection_overview(rates) }} {% for rate in rates %}

{{ rate.description }}

{% endfor %}

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each Human finding once out of all imported Human findings.

From c95b1f9c2f956ac762a26002352718e4ba2acb2b Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:52:38 -0400 Subject: [PATCH 088/126] feat(evaluator): search and filter embedded matching evidence --- README.md | 12 ++ pyproject.toml | 2 +- tests/report_interactions.cjs | 145 ++++++++++++++++++ tests/test_evaluation.py | 2 +- tests/test_report_interactions.py | 37 +++++ vision_aid/evaluation/templates/controls.html | 28 ++++ vision_aid/evaluation/templates/filters.js | 90 +++++++++++ vision_aid/evaluation/templates/full.html | 16 +- vision_aid/evaluation/templates/page.html | 10 +- vision_aid/evaluation/templates/report.css | 7 + 10 files changed, 336 insertions(+), 13 deletions(-) create mode 100644 tests/report_interactions.cjs create mode 100644 tests/test_report_interactions.py create mode 100644 vision_aid/evaluation/templates/controls.html create mode 100644 vision_aid/evaluation/templates/filters.js diff --git a/README.md b/README.md index 68db031..1caf152 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,18 @@ evaluation `report-all` command adds `report.html` at the parent and each child run root. Reports can regenerate from saved reviewed evidence without model calls; both commands require explicit reviewer attribution. +Reports embed all evidence, styling and controls for offline use. Search includes +collapsed Human and matched tool evidence, recorded explanations, rationale and +notes. Each matching excerpt has a Show match control that opens its evidence +panel. Full reports can filter by page; selected-page outcomes appear beside the +combined outcomes without changing the complete-run detection results. + +The LLM and Programmatic missed filters include only findings eligible for that +source. Caught filters include cross-catches by that source. Clear filters restores +all original findings. Expand all visible evidence and Collapse all visible +evidence affect the filtered rows and their visible page panels. With JavaScript +disabled, all findings and native evidence disclosures remain available. + New runs and reporting use only this layout. Existing run directories remain untouched and require manual reorganization before the updated `report` command can read them. Move their supporting files into `artifacts/`, including the diff --git a/pyproject.toml b/pyproject.toml index 87b54ac..3f49825 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,5 +38,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.json", "templates/*.html", "templates/*.css"] +"vision_aid.evaluation" = ["pricing.json", "templates/*.html", "templates/*.css", "templates/*.js"] "processing_scripts.llm" = ["*.txt"] diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs new file mode 100644 index 0000000..6407627 --- /dev/null +++ b/tests/report_interactions.cjs @@ -0,0 +1,145 @@ +/* Browser checks observe CLI-generated documents, never a template/helper API. */ +const assert = require('node:assert/strict'); +const path = require('node:path'); +const {pathToFileURL} = require('node:url'); +const {chromium} = require(path.join(process.env.REPORT_BROWSER_MODULES, 'playwright')); +const [behavior, fullFile, pageFile] = process.argv.slice(2); +(async () => { + const browser = await chromium.launch({headless: true, executablePath: process.env.REPORT_BROWSER_EXECUTABLE}); + try { + for (const filename of [fullFile, pageFile]) { + const context = await browser.newContext({offline: true}); + const page = await context.newPage(); + page.setDefaultTimeout(3000); + const requests = []; + page.on('request', request => {if (!request.url().startsWith('file:')) requests.push(request.url());}); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + await page.goto(pathToFileURL(filename).href); + const rows = page.locator('#findings > tbody > tr'); + const visibleTitles = () => page.locator('#findings > tbody > tr:visible > th').allTextContents(); + if (behavior === 'search') { + assert.equal(await rows.locator('details[open]').count(), 0); + const search = page.getByLabel('Search findings and evidence'); + await search.fill('Complete Global on home'); + assert.deepEqual(await visibleTitles(), ['Shared defect']); + assert.equal(await search.evaluate(e => document.activeElement === e), true); + const match = page.getByRole('button', {name: /Show match in.*Complete Global on home/}); + assert.equal(await match.count(), 1); + assert.match(await match.textContent(), /https:\/\/example.test\//); + await match.click(); + const global = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); + assert.equal(await global.locator('details[open]').count(), filename === fullFile ? 2 : 1); + if (filename === fullFile) { + await search.fill('Complete Global on another page'); + const pageMatches = page.getByRole('button', {name: /Show match in.*Complete Global on another page/}); + assert.equal(await pageMatches.count(), 2); + await pageMatches.nth(0).click(); + await pageMatches.nth(1).click(); + assert.equal(await global.locator('details details[open]').count(), 3); + } + await search.fill('Synthetic failure'); + assert.deepEqual(await visibleTitles(), filename === fullFile ? ['First local', 'Second local', 'Shared defect'] : ['First local', 'Second local']); + assert.ok(await page.getByRole('button', {name: /Show match in.*Synthetic failure/}).count()); + await search.fill('Original evidence'); + assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 5 : 4); + assert.ok(await page.getByRole('button', {name: /Show match in Original Human finding/}).count()); + await search.fill('does-not-appear-in-any-evidence'); + assert.equal(await rows.filter({visible: true}).count(), 0); + assert.match(await page.getByRole('status').textContent(), /No matching findings/); + await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); + assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 8 : 7); + } + if (behavior === 'filters') { + const performance = await page.locator('#performance').textContent(); + const outcome = page.getByLabel('Detection outcome', {exact: true}); + const count = page.getByRole('status'); + await outcome.selectOption({label: 'LLM missed (LLM eligible)'}); + assert.deepEqual(await visibleTitles(), filename === fullFile ? ['Contact local', 'Partial across pages'] : ['Shared defect', 'Partial across pages']); + await outcome.focus(); + await page.keyboard.press('Home'); + await page.keyboard.press('End'); + assert.equal(await outcome.evaluate(e => document.activeElement === e), true); + // Explicit catches include cross-catches irrespective of expected source. + await outcome.selectOption({label: 'LLM caught (any eligibility)'}); + assert.deepEqual(await visibleTitles(), filename === fullFile ? ['First local', 'Second local', 'Shared defect'] : ['First local', 'Second local']); + assert.match(await count.textContent(), /Filtered findings: (2|3) of (7|8)/); + assert.equal(await count.getAttribute('aria-live'), 'polite'); + if (filename === fullFile) { + const pageFilter = page.getByLabel('Page', {exact: true}); + await pageFilter.selectOption({label: 'https://example.test/'}); + await outcome.selectOption({label: 'LLM missed (LLM eligible)'}); + assert.deepEqual(await visibleTitles(), ['Shared defect', 'Partial across pages']); + const shared = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); + const selected = shared.getByText(/Selected page:/); + assert.match(await selected.textContent(), /https:\/\/example.test\/.*LLM: Not caught.*Programmatic checks: Caught/); + assert.match(await shared.locator(':scope > td').nth(3).textContent(), /Caught/); + await page.getByLabel('Search findings and evidence').fill('Complete Global on another page'); + assert.deepEqual(await visibleTitles(), []); + await page.getByLabel('Search findings and evidence').fill('Complete Global on home'); + assert.deepEqual(await visibleTitles(), ['Shared defect']); + await page.getByLabel('Search findings and evidence').fill(''); + await pageFilter.selectOption({label: 'https://example.test/contact/'}); + assert.deepEqual(await visibleTitles(), ['Contact local', 'Partial across pages']); + await outcome.selectOption({label: 'Programmatic missed (Programmatic eligible)'}); + assert.deepEqual(await visibleTitles(), ['Eligible uncaught']); + await pageFilter.selectOption({label: 'https://example.test/'}); + assert.deepEqual(await visibleTitles(), ['Second local', 'Joint catch']); + await outcome.selectOption({label: 'Not testable from the page file'}); + assert.deepEqual(await visibleTitles(), ['Shared second defect', 'Eligible uncaught']); + } + assert.equal(await page.locator('#performance').textContent(), performance); + await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); + assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 8 : 7); + assert.equal(await page.getByRole('button', {name: 'Clear filters', exact: true}).evaluate(e => document.activeElement === e), true); + } + if (behavior === 'disclosures') { + const search = page.getByLabel('Search findings and evidence'); + await search.fill('Shared defect'); + if (filename === fullFile) await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); + const expand = page.getByRole('button', {name: 'Expand all visible evidence', exact: true}); + await expand.focus(); + await page.keyboard.press('Enter'); + assert.equal(await expand.evaluate(e => document.activeElement === e), true); + assert.equal(await rows.locator('details[open]').count(), filename === fullFile ? 2 : 1); + // Clearing expands neither the formerly hidden rows nor other page panels. + await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); + assert.equal(await rows.locator('details[open]').count(), filename === fullFile ? 2 : 1); + await page.getByRole('button', {name: 'Collapse all visible evidence', exact: true}).click(); + assert.equal(await rows.locator('details[open]').count(), 0); + await expand.click(); + assert.equal(await rows.locator('details:not([open])').count(), 0); + if (filename === fullFile) { + const global = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); + const summaries = global.locator('details details > summary'); + await summaries.nth(0).click(); + assert.equal(await global.locator('details details[open]').count(), 2); + await summaries.nth(0).click(); + assert.equal(await global.locator('details details[open]').count(), 3); + } + } + assert.deepEqual(requests, []); + assert.deepEqual(errors, []); + await context.close(); + if (behavior === 'disclosures') { + const nojs = await browser.newContext({offline: true, javaScriptEnabled: false}); + const plain = await nojs.newPage(); + await plain.goto(pathToFileURL(filename).href); + assert.equal(await plain.getByLabel('Search findings and evidence').isVisible(), false); + assert.equal(await plain.locator('#findings > tbody > tr:visible').count(), filename === fullFile ? 8 : 7); + const global = plain.locator('#findings > tbody > tr').filter({has: plain.getByRole('rowheader', {name: 'Shared defect', exact: true})}); + await global.locator('summary').first().focus(); + await plain.keyboard.press('Enter'); + assert.equal(await global.locator('details[open]').count(), 1); + if (filename === fullFile) { + for (const n of [0, 1]) { + await global.locator('details details > summary').nth(n).focus(); + await plain.keyboard.press('Enter'); + } + assert.equal(await global.locator('details[open]').count(), 3); + } + await nojs.close(); + } + } + } finally { await browser.close(); } +})().catch(error => {console.error(error); process.exitCode = 1;}); diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 4cb729c..ba4e89c 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -859,7 +859,7 @@ def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monke text = evidence.get_text(" ", strip=True) assert "Editable evidence replacement" not in text and "Edited title" not in text assert "Vague link\nfor everyone" in text and "Unclear link" in text - assert hostile in text and not document.select("script, img, [onerror]") + assert hostile in text and not document.select("header script, #findings script, img, [onerror]") assert "Synthetic checklist 42" in text assert all(item["finding_id"] in text for item in findings[:2]) assert "Together they fully cover the link" in text and "Footer" in text diff --git a/tests/test_report_interactions.py b/tests/test_report_interactions.py new file mode 100644 index 0000000..773a6d3 --- /dev/null +++ b/tests/test_report_interactions.py @@ -0,0 +1,37 @@ +"""Exercise the stakeholder's standalone report through the CLI and browser. + +Set REPORT_BROWSER_MODULES to a directory containing Playwright, and +REPORT_BROWSER_EXECUTABLE to a Chromium executable to run these optional checks. +""" + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +from test_full_report import completed_review, full_workspace, report_all + +# Re-export the synthetic saved-evidence fixtures for pytest discovery. +__all__ = ["completed_review", "full_workspace"] + + +@pytest.mark.parametrize("behavior", ["search", "filters", "disclosures"]) +def test_offline_report_interactions(completed_review, tmp_path, behavior): + """Users can investigate saved evidence without resources beside the HTML.""" + modules = os.environ.get("REPORT_BROWSER_MODULES") + executable = os.environ.get("REPORT_BROWSER_EXECUTABLE") + if not modules or not executable or not shutil.which("node"): + pytest.skip("Set REPORT_BROWSER_MODULES and REPORT_BROWSER_EXECUTABLE for browser checks") + run, manifest = completed_review + assert report_all(run) == 0 + standalone = tmp_path / "standalone" + standalone.mkdir() + parent = standalone / "full.html" + child = standalone / "page.html" + shutil.copyfile(run / "report.html", parent) + shutil.copyfile(run / manifest["pages"][0]["run_dir"] / "report.html", child) + subprocess.run( + ["node", str(Path(__file__).with_name("report_interactions.cjs")), + behavior, str(parent), str(child)], check=True, timeout=60, + ) diff --git a/vision_aid/evaluation/templates/controls.html b/vision_aid/evaluation/templates/controls.html new file mode 100644 index 0000000..db6b946 --- /dev/null +++ b/vision_aid/evaluation/templates/controls.html @@ -0,0 +1,28 @@ + diff --git a/vision_aid/evaluation/templates/filters.js b/vision_aid/evaluation/templates/filters.js new file mode 100644 index 0000000..fc7dc31 --- /dev/null +++ b/vision_aid/evaluation/templates/filters.js @@ -0,0 +1,90 @@ +/* All evidence is already in this document; no network or runtime dependencies. */ +(() => { + 'use strict'; + const controls = document.getElementById('finding-controls'); + const search = document.getElementById('finding-search'); + const pageFilter = document.getElementById('finding-page'); + const outcome = document.getElementById('finding-outcome'); + const status = document.getElementById('finding-count'); + const normalize = text => text.replace(/\s+/g, ' ').trim().toLocaleLowerCase(); + const rows = [...document.querySelectorAll('#findings > tbody > tr')].map(row => { + const locations = [...row.querySelectorAll('[data-evidence-location]')].map(element => ({ + element, text: element.textContent.replace(/\s+/g, ' ').trim(), + label: element.dataset.evidenceLocation, + page: element.closest('[data-page-decision]')?.dataset.page, + })); + const overview = [...row.children].slice(0, -1).map(cell => cell.textContent).join(' '); + return {row, locations, decisions: [...row.querySelectorAll('[data-page-decision]')], + selected: row.querySelector('[data-selected-page-result]'), + overview: normalize(overview), matches: row.querySelector('[data-search-matches]')}; + }); + function matchesOutcome(data) { + const llm = data.llm === 'Caught'; + const programmatic = data.programmatic === 'Caught'; + switch (outcome.value) { + case 'llm-caught': return llm; + case 'llm-missed': return data.expected === 'LLM' && !llm; + case 'programmatic-caught': return programmatic; + case 'programmatic-missed': return data.expected === 'Programmatic checks' && !programmatic; + case 'overall-caught': return llm || programmatic; + case 'overall-missed': return !llm && !programmatic; + case 'not-testable': return data.expected === 'Not testable from the page file'; + default: return true; + } + } + function update() { + const query = normalize(search.value); + const selectedPage = pageFilter?.value || ''; + let count = 0; + for (const item of rows) { + const pageDecision = selectedPage ? item.decisions.find(panel => panel.dataset.page === selectedPage) : null; + const matches = query ? item.locations.filter(location => + (!selectedPage || !location.page || location.page === selectedPage) && normalize(location.text).includes(query)) : []; + item.row.hidden = Boolean((selectedPage && !pageDecision) || + !matchesOutcome(pageDecision?.dataset || item.row.dataset) || + (query && !matches.length && !item.overview.includes(query))); + item.selected.hidden = !selectedPage || !pageDecision; + item.selected.textContent = pageDecision ? `Selected page: ${selectedPage}. Expected source: ${pageDecision.dataset.expected}. LLM: ${pageDecision.dataset.llm}. Programmatic checks: ${pageDecision.dataset.programmatic}. Combined results remain in the overview columns.` : ''; + for (const panel of item.decisions) panel.hidden = Boolean(selectedPage && panel.dataset.page !== selectedPage); + for (const tr of item.row.querySelectorAll('[data-page-table-row]')) tr.hidden = Boolean(selectedPage && tr.dataset.pageTableRow !== selectedPage); + item.matches.replaceChildren(); + if (!item.row.hidden) count++; + if (item.row.hidden || !query) continue; + for (const match of matches) { + const start = Math.max(0, normalize(match.text).indexOf(query) - 35); + const excerpt = match.text.slice(start, start + Math.max(160, query.length + 35)); + const button = document.createElement('button'); + button.type = 'button'; + button.textContent = `Show match in ${match.label}: ${start ? '…' : ''}${excerpt}`; + button.addEventListener('click', () => { + const panel = match.element.closest('details'); + for (let ancestor = panel; ancestor; ancestor = ancestor.parentElement.closest('details')) ancestor.open = true; + panel.querySelector('summary').focus(); + }); + item.matches.append(button); + } + } + status.textContent = `Filtered findings: ${count} of ${rows.length} original Human findings.${count ? '' : ' No matching findings.'} Complete-run detection results are unchanged.`; + } + search.addEventListener('input', update); + pageFilter?.addEventListener('change', update); + outcome.addEventListener('change', update); + document.getElementById('clear-filters').addEventListener('click', () => { + search.value = ''; + if (pageFilter) pageFilter.value = ''; + outcome.value = 'all'; + update(); + }); + for (const [id, open] of [['expand-visible', true], ['collapse-visible', false]]) { + document.getElementById(id).addEventListener('click', () => { + for (const {row} of rows) { + if (row.hidden) continue; + for (const panel of row.querySelectorAll('details')) { + if (!panel.closest('[hidden]')) panel.open = open; + } + } + }); + } + update(); + controls.hidden = false; +})(); diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html index 8e0364f..8620670 100644 --- a/vision_aid/evaluation/templates/full.html +++ b/vision_aid/evaluation/templates/full.html @@ -26,29 +26,30 @@

Detection results

Human findings

Each original Human finding appears once in source order. Evidence starts collapsed. Open View evidence for the original Human finding and its review decisions. Global findings include every page decision, with independently expandable page evidence.

+{% include 'controls.html' %}
{% for row in rows %} - + - {% endfor %} @@ -69,5 +70,6 @@

Evaluated pages in source order

{% for page in pages %}
{{ page.manifest.source_url }}
Run: {{ page.manifest.run_id }}; Benchmark snapshot SHA-256: {{ page.manifest.snapshot_sha256 }}. Unreadable responses: {{ page.unreadable }} of {{ page.prompts }} LLM checks{% if page.unreadable %}; {{ page.failed_checks }}{% endif %}.
{% endfor %}
+ diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html index 25184c4..cb5ee59 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html @@ -25,18 +25,19 @@

Detection results

Human findings

Evidence starts collapsed. Open any View evidence control to read the original Human finding and the review decision.

+{% include 'controls.html' %}
Original Human findings and combined detection outcomes
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksEvidence
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
-{{ human_evidence(row) }} +
View evidence: {{ row.human.problem }}
+
{{ human_evidence(row) }}
{% if row.human.page_scope == 'global' %}

Combined result

Expected source: {{ row.expected }}. Combined LLM: {{ row.llm_outcome }}. Combined Programmatic checks: {{ row.programmatic_outcome }}. Any complete catch on one page earns combined credit once for that source; partial evidence across pages earns none.

-{% for page in row.pages %}{% endfor %} +{% for page in row.pages %}{% endfor %}
Every page decision for {{ row.human.problem }}
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
-{% for page in row.pages %}
View page evidence: {{ page.source_url }} for {{ row.human.problem }} +{% for page in row.pages %}
View page evidence: {{ page.source_url }} for {{ row.human.problem }}

Page: {{ page.source_url }}; run {{ page.run_id }}. LLM: {{ page.llm_outcome }}. Programmatic checks: {{ page.programmatic_outcome }}.

-{{ decision_evidence(page) }} +
{{ decision_evidence(page) }}
{% endfor %} {% else %} -{% for page in row.pages %}

Assigned page: {{ page.source_url }}; run {{ page.run_id }}.

{{ decision_evidence(page) }}{% endfor %} +{% for page in row.pages %}

Assigned page: {{ page.source_url }}; run {{ page.run_id }}.

{{ decision_evidence(page) }}
{% endfor %} {% endif %}
{% for row in rows %} - + - {% endfor %} @@ -51,5 +52,6 @@

Human findings

Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
Benchmark snapshot SHA-256
{{ manifest.snapshot_sha256 }}
Saved pricing version
{{ pricing_version }}
Saved pricing SHA-256
{{ pricing_identity }}
{% for source in pricing_sources %}
Pricing source address
{{ source }}
{% endfor %}
+ diff --git a/vision_aid/evaluation/templates/report.css b/vision_aid/evaluation/templates/report.css index de7b90c..ea967e9 100644 --- a/vision_aid/evaluation/templates/report.css +++ b/vision_aid/evaluation/templates/report.css @@ -23,3 +23,10 @@ dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } .matched { border-left: 3px solid #687f93; padding-left: 1rem; margin-block: 1rem; } @media (max-width: 40rem) { body { padding: .8rem; } .evidence { min-width: 13rem; } } @media print { body { max-width: none; } .skip, nav { display: none; } .table-scroll { overflow: visible; } } + +#finding-controls fieldset { border: 1px solid #8a99a6; padding: 1rem; } +#finding-controls label { display: block; font-weight: 650; margin-top: .8rem; } +#finding-controls input, #finding-controls select { display: block; width: 100%; max-width: 42rem; font: inherit; padding: .5rem; color: inherit; background: #fff; border: 1px solid #687f93; } +#finding-controls button, [data-search-matches] button { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } +[data-search-matches] button { text-align: left; overflow-wrap: anywhere; max-width: 100%; } +[data-selected-page-result] { border-left: 3px solid #687f93; padding-left: .7rem; } From 1b09dc6e743b67b45898c91fad24aaf9d20f8e87 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 10:54:20 -0400 Subject: [PATCH 089/126] docs(evaluator): record offline filtering verification --- docs/evaluation-report-verification.md | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index abe3131..be94c34 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -135,3 +135,58 @@ first native disclosure in both scopes. These checks do not replace actual screen-reader reading, manual keyboard inspection or actual browser zoom. Those checks remain outstanding under the assistive-technology verification ticket, and no WCAG conformance claim is made. + +## Search, filters and disclosures, 2026-10-15 + +Both report commands now embed search, source-aware outcome filters, clear and +visible-evidence expand/collapse controls. Full reports also filter by page. +These controls use only evidence already embedded in the document. Detection +results and charts continue to describe the complete run. + +- After merging the chart work, the complete suite passed with 143 tests, + including three browser tests. Ruff for `vision_aid/evaluation` and `tests`, + and `git diff --check`, passed. +- The browser tests generate reports through the public `report-all` CLI using + synthetic saved reviews and replayed provider responses. They copy parent and + child HTML into a directory without run evidence, then open the files offline. + They cover initially hidden Human and tool evidence and rationale, matching + page/excerpt controls, independently opened page matches, combined versus + selected-page Global outcomes, cross-catches, eligible source misses, + not-testable results, combined page/outcome/text filters, empty results, + clearing, stable complete-run results, live counts, preserved filter focus, + visible-only expand/collapse and native disclosures with JavaScript disabled. +- A wheel built with `uv build --wheel` includes `controls.html` and `filters.js`. + A fresh environment at `/tmp/va-dat-ticket04-installed` invoked both commands + from `/tmp`. The same three browser checks passed against the installed + package's copied standalone outputs at + `/tmp/va-dat-ticket04-installed-standalone/`. +- The saved 17-page evaluation regenerated from a separate disposable copy at + `/tmp/va-dat-ticket04-runs/20261011T015629-880f3e7c`. Its parent was copied alone + to `/tmp/va-dat-ticket04-standalone/report.html`. No saved reviews changed and + no live provider calls were made. +- Chromium 153.0.8010.12 with networking disabled made no HTTP requests for the + saved parent or installed-package synthetic parent and child. All three + documents stayed within a 320 CSS-pixel viewport, with the findings tables + retaining their scroll regions. Native disclosures opened by keyboard with + JavaScript disabled. axe-core 4.14.0 reported no violations or incomplete + checks against WCAG 2 A/AA and 2.1/2.2 AA tags, both initially and after + searching, selecting a page where available, and expanding visible evidence. + +To repeat the committed browser checks, install Playwright in a local Node +module directory and provide a compatible Chromium executable, then run: + +```sh +REPORT_BROWSER_MODULES=/path/to/node_modules \ +REPORT_BROWSER_EXECUTABLE=/path/to/chromium \ +uv run pytest tests/test_report_interactions.py -q +``` + +The tests skip when these environment variables or Node are absent. They require +no browser test service or runtime network connection. In environments with +non-system browser libraries, also set `LD_LIBRARY_PATH` to those libraries. + +These are automated browser checks. Manual keyboard inspection, actual browser +zoom and screen-reader verification remain outstanding, including spoken filter +count announcements. The stakeholder's assistive technology setup remains +unknown. Complete printing is the next implementation ticket; these results do +not establish WCAG conformance. From 7bb039bb8b30bcec14e9429961be2f1f82da6f43 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 11:04:09 -0400 Subject: [PATCH 090/126] feat(evaluator): print complete offline evaluation reports --- README.md | 14 ++++ docs/evaluation-report-verification.md | 69 ++++++++++++++++++ tests/report_interactions.cjs | 87 ++++++++++++++++++++++- tests/test_report_interactions.py | 26 +++++-- vision_aid/evaluation/templates/full.html | 6 +- vision_aid/evaluation/templates/page.html | 6 +- vision_aid/evaluation/templates/print.css | 28 ++++++++ vision_aid/evaluation/templates/print.js | 47 ++++++++++++ 8 files changed, 274 insertions(+), 9 deletions(-) create mode 100644 vision_aid/evaluation/templates/print.css create mode 100644 vision_aid/evaluation/templates/print.js diff --git a/README.md b/README.md index 1920c75..eb11d4e 100644 --- a/README.md +++ b/README.md @@ -772,6 +772,20 @@ all original findings. Expand all visible evidence and Collapse all visible evidence affect the filtered rows and their visible page panels. With JavaScript disabled, all findings and native evidence disclosures remain available. +Use **Print complete report** in either report to print every original Human +finding and every page's matching evidence, even while searching, filtering or +reading collapsed disclosures. It includes charts, exact data, metadata and +scoring explanations. Evidence uses the page width and can continue across +sheets. Interactive controls are omitted. After printing or cancelling, the +report restores your filters, opened disclosures and keyboard focus. This action +works offline and does not change scores or saved reviews. + +The dedicated action is the complete-evidence printing path. Ordinary browser +printing uses the current reading state. Without JavaScript, open the evidence +you need before using the browser's Print command. Printed output and browser PDF +export are not verified accessible PDFs; accessible PDF production is outside +this feature's scope. + New runs and reporting use only this layout. Existing run directories remain untouched and require manual reorganization before the updated `report` command can read them. Move their supporting files into `artifacts/`, including the diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index be94c34..cc5958c 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -190,3 +190,72 @@ zoom and screen-reader verification remain outstanding, including spoken filter count announcements. The stakeholder's assistive technology setup remains unknown. Complete printing is the next implementation ticket; these results do not establish WCAG conformance. + +## Complete printing, 2026-10-15 + +Both generated report scopes expose Print complete report after their embedded +print script initializes. The action reveals every original finding, page +decision, compact page-table row and nested disclosure before invoking browser +printing. It restores hidden states and independently opened disclosures on +`afterprint`, without changing filter values, counts, evidence or scoring. +Keyboard focus and scroll position return when screen media resumes. The same +browser notification handles completion and cancellation. + +Print styling lays each finding out as a full-width block with labelled outcomes; +long evidence can continue across sheets. Charts retain exact labels and their +equivalent data tables, other tables repeat their column headers, and controls, +search excerpts and selected-page filter summaries are omitted. Ordinary browser +printing follows the current reading state. With JavaScript disabled, readers +can open native disclosures before using the browser's Print command; no scripted +print or filter controls are offered. + +- The full suite passed with 144 tests, including four optional browser checks. + Ruff for `vision_aid/evaluation` and `tests`, and `git diff --check`, passed. +- The committed print check uses CLI-generated synthetic parent and child + reports, each copied into its own directory without run evidence or other HTML + files, and opens them through `file:` with networking disabled. It exercises + active text, page and outcome filters, independently opened nested Global + panels, and 250 lines of review notes spanning sheets. Real headless Chromium + A4 PDFs include all matched IDs, saved Human and tool text, rationale, notes, + charts' exact data, metadata and scoring explanations. Basic no-JavaScript + printing includes the manually opened evidence. The check also verifies full + evidence and caption widths and preservation of ordinary-print filtering. +- Completion and cancellation state restoration are exercised with simulated + browser lifecycle events. The PDF export itself also emits Chromium print + lifecycle events. This does not exercise a human completing or cancelling a + native preview dialog; that remains a manual check under ticket 06. +- `uv build --wheel` bundled `print.css` and `print.js`. A fresh environment at + `/tmp/va-dat-ticket05-installed` invoked installed `report-all` and `report` + commands from `/tmp` against disposable synthetic and real saved runs. The + committed print browser check passed against the installed package's + synthetic outputs. No live provider calls were made. +- The real run regenerated in a separate disposable copy at + `/tmp/va-dat-ticket05-runs/20261011T015629-880f3e7c`. Its standalone installed + outputs and temporary PDFs are under + `/tmp/va-dat-ticket05-standalone/real/`, in separate `full` and `page` + directories. Source run evidence and review decisions were not edited. +- Chromium 153.0.8010.12 on Linux exported the complete real Full report as + 273 A4 sheets and its homepage as 35 sheets. `pdftotext` found all 86 original + titles and 187 matched finding IDs in the parent, and all 16 original titles + and 23 matched IDs in the homepage. Comparison of 3,118 parent evidence text + blocks found all ordinary text, with nine private-use icon glyph fragments + containing U+F104 failing exact PDF text extraction. All 426 homepage blocks + matched. Saved evidence is preserved as supplied; these results do not promise + exact font-glyph preservation. Raster inspection checked representative printed + table and full-width evidence pages. +- The real copied documents made no network requests or browser script errors. + Initial-document axe-core 4.14.0 checks with WCAG 2 A/AA and 2.1/2.2 AA tags + reported no violations or incomplete checks. Both retained a 320 CSS-pixel + document width, and native disclosures opened by keyboard without JavaScript. + A separate fully expanded 320-pixel scan reported no violations but left + color-contrast checks incomplete for table cells obscured by their scroll + containers. Initial and targeted expanded checks do not establish coverage + of every possible expanded state. + +Run the browser checks as documented above. The print case additionally requires +Poppler's `pdftotext`; it skips when that executable is unavailable. Only the +Chromium/Linux combination above has been exercised for printing. Native preview +completion/cancellation, other browsers, actual zoom, manual keyboard inspection, +private-use icon appearance and screen-reader reading remain under ticket 06. +Accessible PDF production and claims of WCAG conformance remain outside this +verification. Headless PDF text checks do not establish accessible PDF output. diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs index 6407627..80f4226 100644 --- a/tests/report_interactions.cjs +++ b/tests/report_interactions.cjs @@ -1,6 +1,7 @@ /* Browser checks observe CLI-generated documents, never a template/helper API. */ const assert = require('node:assert/strict'); const path = require('node:path'); +const {execFileSync} = require('node:child_process'); const {pathToFileURL} = require('node:url'); const {chromium} = require(path.join(process.env.REPORT_BROWSER_MODULES, 'playwright')); const [behavior, fullFile, pageFile] = process.argv.slice(2); @@ -118,14 +119,89 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.equal(await global.locator('details details[open]').count(), 3); } } + if (behavior === 'print') { + const print = page.getByRole('button', {name: 'Print complete report', exact: true}); + assert.equal(await print.count(), 1); + // Preserve an uneven mixture of independently opened nested disclosures. + await rows.nth(0).locator('summary').first().click(); + if (filename === fullFile) { + const shared = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); + await shared.locator('summary').first().click(); + await shared.locator('details details > summary').nth(0).click(); + await shared.locator('details details > summary').nth(2).click(); + await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); + } + await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'LLM missed (LLM eligible)'}); + await page.getByLabel('Search findings and evidence').fill('Complete Global on home'); + // Ordinary printing follows the current reading state. + await page.emulateMedia({media: 'print'}); + assert.equal(await rows.filter({visible: true}).count(), 1); + await page.emulateMedia({media: 'screen'}); + const originalEvidence = await page.locator('main').textContent(); + const savedState = () => page.evaluate(() => ({ + hidden: [...document.querySelectorAll('#findings tr, [data-page-decision]')].map(e => e.hidden), + open: [...document.querySelectorAll('details')].map(e => e.open), + values: [...document.querySelectorAll('input, select')].map(e => e.value), + status: document.getElementById('finding-count').textContent, + focused: document.activeElement.id, + })); + // Headless Chromium cannot operate native preview or distinguish its + // completion/cancellation. Simulate those browser lifecycle notifications + // and separately inspect a real Chromium PDF before restoring the state. + await page.evaluate(() => {window.print = () => window.dispatchEvent(new Event('beforeprint'));}); + for (const result of ['completed', 'cancelled']) { + await print.focus(); + const before = await savedState(); + await page.keyboard.press('Enter'); + assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 8 : 7); + assert.equal(await page.locator('#findings > tbody > tr[hidden], [data-page-decision][hidden], [data-page-table-row][hidden]').count(), 0); + assert.equal(await page.locator('details:not([open])').count(), 0); + await page.emulateMedia({media: 'print'}); + assert.equal(await print.isVisible(), false); + assert.equal(await page.locator('#finding-controls').isVisible(), false); + assert.equal(await page.locator('[data-search-matches]:visible, [data-selected-page-result]:visible').count(), 0); + const evidenceWidth = await rows.nth(0).locator('.evidence').evaluate(e => e.getBoundingClientRect().width); + assert.ok(evidenceWidth > 600, `Print evidence uses the page width: ${evidenceWidth}`); + const captionWidth = await page.locator('#findings > caption').evaluate(e => e.getBoundingClientRect().width); + assert.ok(captionWidth >= evidenceWidth * .9, `Print caption uses the page width: ${captionWidth}`); + if (result === 'completed') { + const pdf = path.join(path.dirname(filename), `${path.basename(filename)}.pdf`); + await page.pdf({path: pdf, format: 'A4', printBackground: true}); + const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); + for (const expected of ['Original Human finding', 'Matching rationale', 'Long evidence begins.', 'Long evidence ends.', + 'Complete Global on home', 'Original evidence', 'Detection overview', 'Exact detection overview data', + 'Review decisions by: Combined reviewer', 'Saved pricing version', 'How to read this report']) { + assert.ok(text.includes(expected), `PDF includes ${expected}`); + } + if (filename === fullFile) { + assert.ok(text.includes('Complete Global on another page')); + assert.ok(text.includes('Exact page LLM detection data')); + assert.ok(text.includes('Every page decision for Shared defect')); + } + const identities = await page.locator('dt').filter({hasText: /^Finding ID$/}).evaluateAll(elements => + elements.map(e => e.nextElementSibling.textContent.trim())); + for (const identity of identities) assert.ok(text.includes(identity), `PDF includes matched finding ${identity}`); + const evidence = await page.locator('#findings dd, #findings .matched > p').allTextContents(); + const compactText = text.replace(/\s+/g, ''); + for (const value of evidence) { + assert.ok(compactText.includes(value.replace(/\s+/g, '')), `PDF includes saved evidence: ${value.slice(0, 100)}`); + } + } + await page.emulateMedia({media: 'screen'}); + await page.evaluate(() => window.dispatchEvent(new Event('afterprint'))); + assert.deepEqual(await savedState(), before, `${result} restores reading state`); + assert.equal(await page.locator('main').textContent(), originalEvidence); + } + } assert.deepEqual(requests, []); assert.deepEqual(errors, []); await context.close(); - if (behavior === 'disclosures') { + if (behavior === 'disclosures' || behavior === 'print') { const nojs = await browser.newContext({offline: true, javaScriptEnabled: false}); const plain = await nojs.newPage(); await plain.goto(pathToFileURL(filename).href); assert.equal(await plain.getByLabel('Search findings and evidence').isVisible(), false); + assert.equal(await plain.getByRole('button', {name: 'Print complete report', exact: true}).isVisible(), false); assert.equal(await plain.locator('#findings > tbody > tr:visible').count(), filename === fullFile ? 8 : 7); const global = plain.locator('#findings > tbody > tr').filter({has: plain.getByRole('rowheader', {name: 'Shared defect', exact: true})}); await global.locator('summary').first().focus(); @@ -138,6 +214,15 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); } assert.equal(await global.locator('details[open]').count(), 3); } + if (behavior === 'print') { + const pdf = path.join(path.dirname(filename), `${path.basename(filename)}-nojs.pdf`); + await plain.pdf({path: pdf, format: 'A4'}); + const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); + for (const expected of ['Shared defect', 'Eligibility classification', 'Complete Global on home', + 'Detection overview', 'Exact detection overview data', 'Saved pricing version']) { + assert.ok(text.includes(expected), `Basic no-JavaScript PDF includes ${expected}`); + } + } await nojs.close(); } } diff --git a/tests/test_report_interactions.py b/tests/test_report_interactions.py index 773a6d3..d3770ad 100644 --- a/tests/test_report_interactions.py +++ b/tests/test_report_interactions.py @@ -10,25 +10,43 @@ from pathlib import Path import pytest -from test_full_report import completed_review, full_workspace, report_all +from test_full_report import ( + completed_review, + full_workspace, + report_all, + review_rows, + save_review, +) # Re-export the synthetic saved-evidence fixtures for pytest discovery. __all__ = ["completed_review", "full_workspace"] -@pytest.mark.parametrize("behavior", ["search", "filters", "disclosures"]) +@pytest.mark.parametrize("behavior", ["search", "filters", "disclosures", "print"]) def test_offline_report_interactions(completed_review, tmp_path, behavior): """Users can investigate saved evidence without resources beside the HTML.""" modules = os.environ.get("REPORT_BROWSER_MODULES") executable = os.environ.get("REPORT_BROWSER_EXECUTABLE") if not modules or not executable or not shutil.which("node"): pytest.skip("Set REPORT_BROWSER_MODULES and REPORT_BROWSER_EXECUTABLE for browser checks") + if behavior == "print" and not shutil.which("pdftotext"): + pytest.skip("Install pdftotext for PDF content checks") run, manifest = completed_review + if behavior == "print": + child = run / manifest["pages"][0]["run_dir"] + decisions = review_rows(child) + decisions[0]["review_notes"] = "Long evidence begins. " + " ".join( + f"Saved review line {number}: keyboard evidence remains readable." + for number in range(250) + ) + " Long evidence ends." + save_review(child, decisions) assert report_all(run) == 0 standalone = tmp_path / "standalone" standalone.mkdir() - parent = standalone / "full.html" - child = standalone / "page.html" + parent = standalone / "full" / "report.html" + child = standalone / "page" / "report.html" + parent.parent.mkdir() + child.parent.mkdir() shutil.copyfile(run / "report.html", parent) shutil.copyfile(run / manifest["pages"][0]["run_dir"] / "report.html", child) subprocess.run( diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html index dae4794..30e872d 100644 --- a/vision_aid/evaluation/templates/full.html +++ b/vision_aid/evaluation/templates/full.html @@ -6,7 +6,7 @@ {{ title }}: {{ manifest.run_id }} - + @@ -15,6 +15,7 @@

{{ title }}

This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

Review decisions by: {{ reviewer }}

+
@@ -37,7 +38,7 @@

Human findings

 {% for row in rows %} - + {% for row in rows %} - + -
Human findings and detection outcomes
Issue titleWCAGExpected sourceLLMProgrammatic checksEvidence
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }} +
View evidence: {{ row.human.problem }}
-{{ human_evidence(row) }} -{{ decision_evidence(row) }} +
{{ human_evidence(row) }}
+
{{ decision_evidence(row) }}
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
{{ human_evidence(row) }}
{% if row.human.page_scope == 'global' %} @@ -74,5 +75,6 @@

Evaluated pages in source order

+ diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html index 38a2d81..b2a2422 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html @@ -6,7 +6,7 @@ {{ title }}: {{ manifest.source_url }} - + @@ -15,6 +15,7 @@

{{ title }}

This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

Review decisions by: {{ reviewer }}

+
@@ -35,7 +36,7 @@

Human findings

{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
{{ human_evidence(row) }}
@@ -55,5 +56,6 @@

Human findings

+ diff --git a/vision_aid/evaluation/templates/print.css b/vision_aid/evaluation/templates/print.css new file mode 100644 index 0000000..5fcac7d --- /dev/null +++ b/vision_aid/evaluation/templates/print.css @@ -0,0 +1,28 @@ +#print-complete-report { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin-bottom: 1rem; cursor: pointer; } +@page { margin: 12mm; } +@media print { + body { max-width: none; padding: 0; color: #000; font-size: 11pt; line-height: 1.4; } + .skip, nav, #print-complete-report, #finding-controls, + [data-search-matches], [data-selected-page-result] { display: none !important; } + .table-scroll { overflow: visible; } + table { table-layout: fixed; } + thead { display: table-header-group; } + th, td { padding: .35rem; } + h1, h2, h3, dt, summary, caption { break-after: avoid; } + summary { color: #000; } + summary::marker { content: ''; } + .evidence { min-width: 0; max-width: none; } + /* Evidence can span many sheets. A wide block avoids tiny table columns and + lets long Global decisions split naturally across page boundaries. */ + #findings, #findings > tbody, #findings > tbody > tr, + #findings > tbody > tr > th, #findings > tbody > tr > td { display: block; width: 100%; } + #findings > caption { display: block; width: 100%; } + #findings > thead { display: none; } + #findings > tbody > tr[hidden] { display: none !important; } + #findings > tbody > tr { margin-block: 1.5rem; border-top: 2px solid #687f93; break-inside: auto; } + #findings > tbody > tr > th { font-size: 13pt; break-after: avoid; } + #findings > tbody > tr > th, #findings > tbody > tr > td { border: 0; } + #findings [data-print-label]::before { content: attr(data-print-label) ': '; font-weight: 700; } + #findings details, #findings .evidence, #findings .matched { break-inside: auto; } + #findings summary { overflow-wrap: anywhere; } +} diff --git a/vision_aid/evaluation/templates/print.js b/vision_aid/evaluation/templates/print.js new file mode 100644 index 0000000..2672f51 --- /dev/null +++ b/vision_aid/evaluation/templates/print.js @@ -0,0 +1,47 @@ +/* Native printing uses embedded evidence only; review data and scores stay saved. */ +(() => { + 'use strict'; + const button = document.getElementById('print-complete-report'); + const printMedia = window.matchMedia('print'); + let readingState = null; + let pendingFocus = null; + function restoreFocus() { + if (!pendingFocus || printMedia.matches) return; + const {focused, scrollX, scrollY} = pendingFocus; + pendingFocus = null; + if (focused?.isConnected) focused.focus({preventScroll: true}); + window.scrollTo(scrollX, scrollY); + } + printMedia.addEventListener('change', restoreFocus); + function restore() { + if (!readingState) return; + const {hidden, disclosures, focused, scrollX, scrollY} = readingState; + readingState = null; + for (const [element, value] of hidden) element.hidden = value; + for (const [element, value] of disclosures) element.open = value; + pendingFocus = {focused, scrollX, scrollY}; + restoreFocus(); + } + button.addEventListener('click', () => { + if (readingState) return; + const elements = [...document.querySelectorAll( + '#findings > tbody > tr, [data-page-decision], [data-page-table-row]')]; + const details = [...document.querySelectorAll('details')]; + readingState = { + hidden: elements.map(element => [element, element.hidden]), + disclosures: details.map(element => [element, element.open]), + focused: document.activeElement, scrollX: window.scrollX, scrollY: window.scrollY, + }; + for (const element of elements) element.hidden = false; + for (const element of details) element.open = true; + try { + window.print(); + } catch (error) { + restore(); + throw error; + } + }); + // Browsers send the same notification on completion and cancellation. + window.addEventListener('afterprint', restore); + button.hidden = false; +})(); From 37db71de6542d32b81ee40022f76f89fe7e4c3e9 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 13:13:01 -0400 Subject: [PATCH 091/126] refactor(evaluator): share HTML report document layout --- docs/evaluation-report-verification.md | 37 +++++++++++++++++++++++ vision_aid/evaluation/templates/base.html | 25 +++++++++++++++ vision_aid/evaluation/templates/full.html | 30 ++++-------------- vision_aid/evaluation/templates/page.html | 30 ++++-------------- 4 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 vision_aid/evaluation/templates/base.html diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index cc5958c..f1c69d4 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -259,3 +259,40 @@ completion/cancellation, other browsers, actual zoom, manual keyboard inspection private-use icon appearance and screen-reader reading remain under ticket 06. Accessible PDF production and claims of WCAG conformance remain outside this verification. Headless PDF text checks do not establish accessible PDF output. + +## Integration review, 2026-10-15 + +The spec review found no implementation issues. The standards review identified +duplicated document layout in the individual and Full templates. Both now extend +one shared `base.html` for the document metadata, embedded assets, skip link, +model and reviewer attribution, navigation, print button and scripts. Their +scope-specific evidence, charts and scoring explanations remain in the existing +templates. + +- The complete suite passed with 144 tests, including all four browser cases. + Scoped Ruff for `vision_aid/evaluation` and `tests`, and `git diff --check`, + passed. Existing public CLI and browser tests cover this template refactor; + no tests tied to the inheritance implementation were added. +- A fresh wheel includes `base.html`. Installed commands generated both scopes + from `/tmp`, outside the checkout, against a separate saved-run copy at + `/tmp/va-dat-report-review-runs/20261011T015629-880f3e7c`. Both generated files + match their pre-refactor HTML except for blank lines. No live provider calls + were made. +- Independent final integration inspection confirmed 86 overview rows, 198 page + decisions and 187 matched ID occurrences, with all saved classifications, + rationale and notes retained. Rates remain 28/62 LLM, 18/18 Programmatic and + 46/86 Overall, and saved cached-token accounting is unchanged. SHA-256 checks + confirmed all 342 original source files were unchanged. +- The installed outputs were copied alone to the `full` and `page` directories + under `/tmp/va-dat-report-review-standalone/`. Chromium 153.0.8010.12 opened + both through `file:` with networking disabled and made no HTTP requests. + axe-core 4.14.0 reported no violations or incomplete checks for WCAG 2 A/AA + and 2.1/2.2 AA tags in each initial document. Both document and body widths + stayed at 320 CSS pixels in a 320-pixel viewport. All 205 Full and 16 page + disclosures started collapsed; keyboard Enter opened a native disclosure + with JavaScript disabled in both scopes. These results also matched the + independent integration checks before the layout refactor. + +The manual assistive-technology and native print-preview checks listed above +remain outstanding under ticket 06. These automated checks do not establish +WCAG conformance or accessible PDF output. diff --git a/vision_aid/evaluation/templates/base.html b/vision_aid/evaluation/templates/base.html new file mode 100644 index 0000000..623b51b --- /dev/null +++ b/vision_aid/evaluation/templates/base.html @@ -0,0 +1,25 @@ + + + + + +{% block document_title %}{% endblock %} + + + + +
+

{{ title }}

+

{% block scope %}{% endblock %}

+

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

+

Review decisions by: {{ reviewer }}

+ + +
+
+{% block content %}{% endblock %} +
+ + + + diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html index 30e872d..9042459 100644 --- a/vision_aid/evaluation/templates/full.html +++ b/vision_aid/evaluation/templates/full.html @@ -1,24 +1,10 @@ +{% extends 'base.html' %} {% from 'evidence.html' import human_evidence, decision_evidence %} {% from 'charts.html' import detection_overview, page_comparison %} - - - - - -{{ title }}: {{ manifest.run_id }} - - - - -
-

{{ title }}

-

This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.

-

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

-

Review decisions by: {{ reviewer }}

- - -
-
+ +{% block document_title %}{{ title }}: {{ manifest.run_id }}{% endblock %} +{% block scope %}This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.{% endblock %} +{% block content %}

Detection results

{{ detection_overview(rates) }} @@ -73,8 +59,4 @@

Combined result

Evaluated pages in source order

{% for page in pages %}
{{ page.manifest.source_url }}
Run: {{ page.manifest.run_id }}; Benchmark snapshot SHA-256: {{ page.manifest.snapshot_sha256 }}. Unreadable responses: {{ page.unreadable }} of {{ page.prompts }} LLM checks{% if page.unreadable %}; {{ page.failed_checks }}{% endif %}.
{% endfor %}
-
- - - - +{% endblock %} diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html index b2a2422..034ac45 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html @@ -1,24 +1,10 @@ +{% extends 'base.html' %} {% from 'evidence.html' import human_evidence, decision_evidence %} {% from 'charts.html' import detection_overview, page_comparison %} - - - - - -{{ title }}: {{ manifest.source_url }} - - - - -
-

{{ title }}

-

This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.

-

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

-

Review decisions by: {{ reviewer }}

- - -
-
+ +{% block document_title %}{{ title }}: {{ manifest.source_url }}{% endblock %} +{% block scope %}This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.{% endblock %} +{% block content %}

Detection results

{{ detection_overview(rates) }} @@ -54,8 +40,4 @@

Human findings

{% if unreadable %}

{{ unreadable }} of {{ prompts }} LLM checks returned an Unreadable response that the tool would skip: {{ failed_checks }}.

{% endif %}
Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
Benchmark snapshot SHA-256
{{ manifest.snapshot_sha256 }}
Saved pricing version
{{ pricing_version }}
Saved pricing SHA-256
{{ pricing_identity }}
{% for source in pricing_sources %}
Pricing source address
{{ source }}
{% endfor %}
-
- - - - +{% endblock %} From c66778e475fc498671b18dc700281196e4ea8478 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 13:24:44 -0400 Subject: [PATCH 092/126] refactor(evaluator): suffix report templates with .jinja --- docs/evaluation-report-verification.md | 12 ++++++++++++ pyproject.toml | 2 +- vision_aid/evaluation/html_report.py | 6 +++--- .../templates/{base.html => base.html.jinja} | 6 +++--- .../templates/{charts.css => charts.css.jinja} | 0 .../templates/{charts.html => charts.html.jinja} | 0 .../templates/{controls.html => controls.html.jinja} | 0 .../templates/{evidence.html => evidence.html.jinja} | 0 .../templates/{filters.js => filters.js.jinja} | 0 .../templates/{full.html => full.html.jinja} | 10 +++++----- .../templates/{method.html => method.html.jinja} | 0 .../templates/{page.html => page.html.jinja} | 10 +++++----- .../templates/{print.css => print.css.jinja} | 0 .../templates/{print.js => print.js.jinja} | 0 .../templates/{report.css => report.css.jinja} | 0 15 files changed, 29 insertions(+), 17 deletions(-) rename vision_aid/evaluation/templates/{base.html => base.html.jinja} (80%) rename vision_aid/evaluation/templates/{charts.css => charts.css.jinja} (100%) rename vision_aid/evaluation/templates/{charts.html => charts.html.jinja} (100%) rename vision_aid/evaluation/templates/{controls.html => controls.html.jinja} (100%) rename vision_aid/evaluation/templates/{evidence.html => evidence.html.jinja} (100%) rename vision_aid/evaluation/templates/{filters.js => filters.js.jinja} (100%) rename vision_aid/evaluation/templates/{full.html => full.html.jinja} (96%) rename vision_aid/evaluation/templates/{method.html => method.html.jinja} (100%) rename vision_aid/evaluation/templates/{page.html => page.html.jinja} (93%) rename vision_aid/evaluation/templates/{print.css => print.css.jinja} (100%) rename vision_aid/evaluation/templates/{print.js => print.js.jinja} (100%) rename vision_aid/evaluation/templates/{report.css => report.css.jinja} (100%) diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index f1c69d4..4802fee 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -296,3 +296,15 @@ templates. The manual assistive-technology and native print-preview checks listed above remain outstanding under ticket 06. These automated checks do not establish WCAG conformance or accessible PDF output. + +## Jinja template suffixes, 2026-10-15 + +All 12 report template sources now end in `.jinja`, retaining their content +extensions, such as `base.html.jinja`, `report.css.jinja`, and `filters.js.jinja`. +Template references, package data, and HTML autoescaping use the new names. + +The 144-test suite passed with browser checks enabled, including hostile-evidence +escaping. Scoped Ruff, lock validation, and whitespace checks passed. A fresh +wheel contained all 12 renamed sources, and its installed `report` and +`report-all` commands generated both scopes outside the checkout using a +disposable saved run. No live provider calls were made. diff --git a/pyproject.toml b/pyproject.toml index 3f49825..75eafea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,5 +38,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.json", "templates/*.html", "templates/*.css", "templates/*.js"] +"vision_aid.evaluation" = ["pricing.json", "templates/*.jinja"] "processing_scripts.llm" = ["*.txt"] diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py index 0ed8bfc..3e2dd6f 100644 --- a/vision_aid/evaluation/html_report.py +++ b/vision_aid/evaluation/html_report.py @@ -168,19 +168,19 @@ def write_full_report( reviews: list[PageReview], reviewer: str, ) -> None: """Render every accepted page decision into the standalone parent.""" - _write_html(path, "full.html", full_data(manifest, pooled, reviews, reviewer)) + _write_html(path, "full.html.jinja", full_data(manifest, pooled, reviews, reviewer)) def write_page_report(path: Path, review: PageReview, reviewer: str) -> None: """Publish a standalone individual Evaluation report.""" - _write_html(path, "page.html", page_data(review, reviewer)) + _write_html(path, "page.html.jinja", page_data(review, reviewer)) def _write_html(path: Path, template: str, data: dict[str, Any]) -> None: """Render completely, then atomically replace the report in its directory.""" environment = Environment( loader=PackageLoader("vision_aid.evaluation", "templates"), - autoescape=select_autoescape(("html",)), + autoescape=select_autoescape(("html.jinja",)), undefined=StrictUndefined, ) try: diff --git a/vision_aid/evaluation/templates/base.html b/vision_aid/evaluation/templates/base.html.jinja similarity index 80% rename from vision_aid/evaluation/templates/base.html rename to vision_aid/evaluation/templates/base.html.jinja index 623b51b..3de962c 100644 --- a/vision_aid/evaluation/templates/base.html +++ b/vision_aid/evaluation/templates/base.html.jinja @@ -4,7 +4,7 @@ {% block document_title %}{% endblock %} - + @@ -19,7 +19,7 @@

{{ title }}

{% block content %}{% endblock %}
- - + + diff --git a/vision_aid/evaluation/templates/charts.css b/vision_aid/evaluation/templates/charts.css.jinja similarity index 100% rename from vision_aid/evaluation/templates/charts.css rename to vision_aid/evaluation/templates/charts.css.jinja diff --git a/vision_aid/evaluation/templates/charts.html b/vision_aid/evaluation/templates/charts.html.jinja similarity index 100% rename from vision_aid/evaluation/templates/charts.html rename to vision_aid/evaluation/templates/charts.html.jinja diff --git a/vision_aid/evaluation/templates/controls.html b/vision_aid/evaluation/templates/controls.html.jinja similarity index 100% rename from vision_aid/evaluation/templates/controls.html rename to vision_aid/evaluation/templates/controls.html.jinja diff --git a/vision_aid/evaluation/templates/evidence.html b/vision_aid/evaluation/templates/evidence.html.jinja similarity index 100% rename from vision_aid/evaluation/templates/evidence.html rename to vision_aid/evaluation/templates/evidence.html.jinja diff --git a/vision_aid/evaluation/templates/filters.js b/vision_aid/evaluation/templates/filters.js.jinja similarity index 100% rename from vision_aid/evaluation/templates/filters.js rename to vision_aid/evaluation/templates/filters.js.jinja diff --git a/vision_aid/evaluation/templates/full.html b/vision_aid/evaluation/templates/full.html.jinja similarity index 96% rename from vision_aid/evaluation/templates/full.html rename to vision_aid/evaluation/templates/full.html.jinja index 9042459..cbbef06 100644 --- a/vision_aid/evaluation/templates/full.html +++ b/vision_aid/evaluation/templates/full.html.jinja @@ -1,6 +1,6 @@ -{% extends 'base.html' %} -{% from 'evidence.html' import human_evidence, decision_evidence %} -{% from 'charts.html' import detection_overview, page_comparison %} +{% extends 'base.html.jinja' %} +{% from 'evidence.html.jinja' import human_evidence, decision_evidence %} +{% from 'charts.html.jinja' import detection_overview, page_comparison %} {% block document_title %}{{ title }}: {{ manifest.run_id }}{% endblock %} {% block scope %}This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.{% endblock %} @@ -16,7 +16,7 @@

Detection results

Human findings

Each original Human finding appears once in source order. Evidence starts collapsed. Open View evidence for the original Human finding and its review decisions. Global findings include every page decision, with independently expandable page evidence.

-{% include 'controls.html' %} +{% include 'controls.html.jinja' %}
@@ -45,7 +45,7 @@

Combined result

{% endfor %}
Original Human findings and combined detection outcomes
-{% include 'method.html' %} +{% include 'method.html.jinja' %}

Full evaluation scoring

Eligibility is established independently of detection. If any page is testable for a Global finding, the finding enters that expected source's denominator once. All testable pages must agree on the source. Findings not testable on any page remain in Overall and outside the LLM and Programmatic denominators.

Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged.

diff --git a/vision_aid/evaluation/templates/method.html b/vision_aid/evaluation/templates/method.html.jinja similarity index 100% rename from vision_aid/evaluation/templates/method.html rename to vision_aid/evaluation/templates/method.html.jinja diff --git a/vision_aid/evaluation/templates/page.html b/vision_aid/evaluation/templates/page.html.jinja similarity index 93% rename from vision_aid/evaluation/templates/page.html rename to vision_aid/evaluation/templates/page.html.jinja index 034ac45..072af46 100644 --- a/vision_aid/evaluation/templates/page.html +++ b/vision_aid/evaluation/templates/page.html.jinja @@ -1,6 +1,6 @@ -{% extends 'base.html' %} -{% from 'evidence.html' import human_evidence, decision_evidence %} -{% from 'charts.html' import detection_overview, page_comparison %} +{% extends 'base.html.jinja' %} +{% from 'evidence.html.jinja' import human_evidence, decision_evidence %} +{% from 'charts.html.jinja' import detection_overview, page_comparison %} {% block document_title %}{{ title }}: {{ manifest.source_url }}{% endblock %} {% block scope %}This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.{% endblock %} @@ -14,7 +14,7 @@

Detection results

Human findings

Evidence starts collapsed. Open any View evidence control to read the original Human finding and the review decision.

-{% include 'controls.html' %} +{% include 'controls.html.jinja' %}
@@ -33,7 +33,7 @@

Human findings

Human findings and detection outcomes
{% if not rows %}

No Human findings were imported for this page.

{% endif %}
-{% include 'method.html' %} +{% include 'method.html.jinja' %}

Cost and provenance

Estimated run cost: ${{ cost }}, calculated using provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

diff --git a/vision_aid/evaluation/templates/print.css b/vision_aid/evaluation/templates/print.css.jinja similarity index 100% rename from vision_aid/evaluation/templates/print.css rename to vision_aid/evaluation/templates/print.css.jinja diff --git a/vision_aid/evaluation/templates/print.js b/vision_aid/evaluation/templates/print.js.jinja similarity index 100% rename from vision_aid/evaluation/templates/print.js rename to vision_aid/evaluation/templates/print.js.jinja diff --git a/vision_aid/evaluation/templates/report.css b/vision_aid/evaluation/templates/report.css.jinja similarity index 100% rename from vision_aid/evaluation/templates/report.css rename to vision_aid/evaluation/templates/report.css.jinja From d5c2900f5716091b440f0e83981a26e07ee95ef6 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 13:27:32 -0400 Subject: [PATCH 093/126] refactor(evaluator): separate static report assets from templates --- docs/evaluation-report-verification.md | 15 +++++++++++++++ pyproject.toml | 2 +- .../charts.css.jinja => css/charts.css} | 0 .../{templates/print.css.jinja => css/print.css} | 0 .../report.css.jinja => css/report.css} | 0 vision_aid/evaluation/html_report.py | 15 ++++++++++++++- .../{templates/filters.js.jinja => js/filters.js} | 0 .../{templates/print.js.jinja => js/print.js} | 0 vision_aid/evaluation/templates/base.html.jinja | 6 +++--- 9 files changed, 33 insertions(+), 5 deletions(-) rename vision_aid/evaluation/{templates/charts.css.jinja => css/charts.css} (100%) rename vision_aid/evaluation/{templates/print.css.jinja => css/print.css} (100%) rename vision_aid/evaluation/{templates/report.css.jinja => css/report.css} (100%) rename vision_aid/evaluation/{templates/filters.js.jinja => js/filters.js} (100%) rename vision_aid/evaluation/{templates/print.js.jinja => js/print.js} (100%) diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 4802fee..150f9fd 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -308,3 +308,18 @@ escaping. Scoped Ruff, lock validation, and whitespace checks passed. A fresh wheel contained all 12 renamed sources, and its installed `report` and `report-all` commands generated both scopes outside the checkout using a disposable saved run. No live provider calls were made. + +## Static asset separation, 2026-10-15 + +The seven HTML templates retain `.html.jinja` names under `templates/`. The +two static JavaScript files now use `.js` under `js/`, and the three static +stylesheets use `.css` under `css/`. Python reads these trusted package assets +as literal text and embeds them in the report; Jinja no longer processes them +as templates. Review and model text remain HTML-escaped. + +All 144 tests passed with browser checks enabled, including search, filtering, +printing, offline reading, and hostile-evidence escaping. Scoped Ruff and +whitespace checks passed. A fresh wheel contained seven HTML templates, two +JavaScript assets, and three CSS assets in their separate folders. Its installed +`report` and `report-all` commands generated both scopes outside the checkout +from a disposable saved run, with no live provider calls. diff --git a/pyproject.toml b/pyproject.toml index 75eafea..87caccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,5 +38,5 @@ include = ["vision_aid*", "processing_scripts*", "entry_points*"] namespaces = true [tool.setuptools.package-data] -"vision_aid.evaluation" = ["pricing.json", "templates/*.jinja"] +"vision_aid.evaluation" = ["pricing.json", "templates/*.html.jinja", "css/*.css", "js/*.js"] "processing_scripts.llm" = ["*.txt"] diff --git a/vision_aid/evaluation/templates/charts.css.jinja b/vision_aid/evaluation/css/charts.css similarity index 100% rename from vision_aid/evaluation/templates/charts.css.jinja rename to vision_aid/evaluation/css/charts.css diff --git a/vision_aid/evaluation/templates/print.css.jinja b/vision_aid/evaluation/css/print.css similarity index 100% rename from vision_aid/evaluation/templates/print.css.jinja rename to vision_aid/evaluation/css/print.css diff --git a/vision_aid/evaluation/templates/report.css.jinja b/vision_aid/evaluation/css/report.css similarity index 100% rename from vision_aid/evaluation/templates/report.css.jinja rename to vision_aid/evaluation/css/report.css diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py index 3e2dd6f..922e8c0 100644 --- a/vision_aid/evaluation/html_report.py +++ b/vision_aid/evaluation/html_report.py @@ -5,6 +5,7 @@ import os import tempfile from decimal import Decimal +from importlib.resources import files from pathlib import Path from typing import TYPE_CHECKING, Any @@ -183,8 +184,20 @@ def _write_html(path: Path, template: str, data: dict[str, Any]) -> None: autoescape=select_autoescape(("html.jinja",)), undefined=StrictUndefined, ) + # Only trusted package assets bypass escaping; review data stays escaped. + assets = files("vision_aid.evaluation") + styles = "".join( + assets.joinpath("css", name).read_text(encoding="utf-8") + for name in ("report.css", "charts.css", "print.css") + ) + scripts = { + name: assets.joinpath("js", f"{name}.js").read_text(encoding="utf-8") + for name in ("filters", "print") + } try: - content = environment.get_template(template).render(**data) + content = environment.get_template(template).render( + **data, styles=styles, scripts=scripts, + ) except TemplateError as error: raise ValueError(f"Cannot render complete report: {error}") from error temporary = None diff --git a/vision_aid/evaluation/templates/filters.js.jinja b/vision_aid/evaluation/js/filters.js similarity index 100% rename from vision_aid/evaluation/templates/filters.js.jinja rename to vision_aid/evaluation/js/filters.js diff --git a/vision_aid/evaluation/templates/print.js.jinja b/vision_aid/evaluation/js/print.js similarity index 100% rename from vision_aid/evaluation/templates/print.js.jinja rename to vision_aid/evaluation/js/print.js diff --git a/vision_aid/evaluation/templates/base.html.jinja b/vision_aid/evaluation/templates/base.html.jinja index 3de962c..69dbd4d 100644 --- a/vision_aid/evaluation/templates/base.html.jinja +++ b/vision_aid/evaluation/templates/base.html.jinja @@ -4,7 +4,7 @@ {% block document_title %}{% endblock %} - + @@ -19,7 +19,7 @@
{% block content %}{% endblock %}
- - + + From d3b7b6a3b74b520a29bd0614282d6143d5966abd Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 14:58:15 -0400 Subject: [PATCH 094/126] docs: define self-contained evaluation reports --- CONTEXT.md | 3 +++ docs/adr/0004-make-evaluation-reports-self-contained.md | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 docs/adr/0004-make-evaluation-reports-self-contained.md diff --git a/CONTEXT.md b/CONTEXT.md index 281e2d1..4e46056 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -28,6 +28,9 @@ _Avoid_: Execution plan, report bundle. An evaluation of every page listed in the Human audit with one selected model held fixed throughout. It includes results for each page and combined detection rates based on Human finding counts. _Avoid_: Site crawl, model comparison. +**Evaluation report**: +The reader-facing account of an Evaluation run or a Full evaluation, explaining its detection rates and the evidence for its Match decisions. A Full evaluation's report includes the supporting page-level decisions for every original Human finding. + **LLM**: The part of the tool that asks a large language model to judge the page for accessibility problems. _Avoid_: Audit model, AI audit. diff --git a/docs/adr/0004-make-evaluation-reports-self-contained.md b/docs/adr/0004-make-evaluation-reports-self-contained.md new file mode 100644 index 0000000..bb380bf --- /dev/null +++ b/docs/adr/0004-make-evaluation-reports-self-contained.md @@ -0,0 +1,3 @@ +# Make evaluation reports self-contained + +Evaluation reports are offline HTML documents for Vision Aid stakeholders, including readers with visual impairments. A Full evaluation's report contains the results and evidence for every Match decision without requiring child reports or external resources. This duplicates some page-level content but lets readers receive one complete document; summary charts and tables keep results visible while accessible disclosure controls reveal detailed evidence on request. From 7ae314e59594784c088a24a045e7aee8ab79acbd Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 16:15:31 -0400 Subject: [PATCH 095/126] feat(reports): clarify findings and highlight cost and detection Use nested page decision details in full and single-page reports. Show saved token usage and model prices beside the estimated run cost, simplify detection charts, and normalize report headings and model labels. Update report and browser checks for the revised layout. --- tests/report_interactions.cjs | 42 ++++--- tests/test_evaluation.py | 106 ++++++++++------- tests/test_full_report.py | 91 ++++++++------- vision_aid/evaluation/css/charts.css | 19 +++- vision_aid/evaluation/html_report.py | 107 +++++++++++------- .../evaluation/templates/base.html.jinja | 8 +- .../evaluation/templates/charts.html.jinja | 38 ++++--- .../evaluation/templates/evidence.html.jinja | 12 +- .../evaluation/templates/full.html.jinja | 34 ++---- .../evaluation/templates/page.html.jinja | 26 ++--- 10 files changed, 277 insertions(+), 206 deletions(-) diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs index 80f4226..0e3559c 100644 --- a/tests/report_interactions.cjs +++ b/tests/report_interactions.cjs @@ -18,6 +18,16 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); page.on('pageerror', error => errors.push(error.message)); await page.goto(pathToFileURL(filename).href); const rows = page.locator('#findings > tbody > tr'); + const cost = page.locator('#performance .run-cost'); + assert.equal(await cost.isVisible(), true); + assert.match(await cost.textContent(), /Estimated run cost:\$0\.\d{6} USD/); + assert.equal(await page.locator('#provenance, a[href="#provenance"]').count(), 0); + assert.ok((await cost.boundingBox()).y < (await page.locator('.detection-chart').first().boundingBox()).y); + await page.setViewportSize({width: 320, height: 800}); + const costBox = await cost.boundingBox(); + assert.ok(costBox.x >= 0 && costBox.x + costBox.width <= 320); + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= 320)); + await page.setViewportSize({width: 1280, height: 900}); const visibleTitles = () => page.locator('#findings > tbody > tr:visible > th').allTextContents(); if (behavior === 'search') { assert.equal(await rows.locator('details[open]').count(), 0); @@ -30,7 +40,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.match(await match.textContent(), /https:\/\/example.test\//); await match.click(); const global = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); - assert.equal(await global.locator('details[open]').count(), filename === fullFile ? 2 : 1); + assert.equal(await global.locator('details[open]').count(), 2); if (filename === fullFile) { await search.fill('Complete Global on another page'); const pageMatches = page.getByRole('button', {name: /Show match in.*Complete Global on another page/}); @@ -102,12 +112,20 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await expand.focus(); await page.keyboard.press('Enter'); assert.equal(await expand.evaluate(e => document.activeElement === e), true); - assert.equal(await rows.locator('details[open]').count(), filename === fullFile ? 2 : 1); + assert.equal(await rows.locator('details[open]').count(), 2); // Clearing expands neither the formerly hidden rows nor other page panels. await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); - assert.equal(await rows.locator('details[open]').count(), filename === fullFile ? 2 : 1); + assert.equal(await rows.locator('details[open]').count(), 2); await page.getByRole('button', {name: 'Collapse all visible evidence', exact: true}).click(); assert.equal(await rows.locator('details[open]').count(), 0); + const local = rows.filter({has: page.getByRole('rowheader', {name: 'First local', exact: true})}); + await local.locator('summary').first().click(); + assert.equal(await local.locator('.matched').first().isVisible(), false); + const localPageSummary = local.locator('details details > summary'); + assert.equal(await localPageSummary.count(), 1); + await localPageSummary.focus(); + await page.keyboard.press('Enter'); + assert.equal(await local.locator('.matched').first().isVisible(), true); await expand.click(); assert.equal(await rows.locator('details:not([open])').count(), 0); if (filename === fullFile) { @@ -169,13 +187,13 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await page.pdf({path: pdf, format: 'A4', printBackground: true}); const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); for (const expected of ['Original Human finding', 'Matching rationale', 'Long evidence begins.', 'Long evidence ends.', - 'Complete Global on home', 'Original evidence', 'Detection overview', 'Exact detection overview data', - 'Review decisions by: Combined reviewer', 'Saved pricing version', 'How to read this report']) { + 'Complete Global on home', 'Original evidence', 'Detection overview', + 'Review decisions by: Combined reviewer', 'Estimated run cost:', await cost.locator('.cost-value').textContent(), 'How to read this report']) { assert.ok(text.includes(expected), `PDF includes ${expected}`); } if (filename === fullFile) { assert.ok(text.includes('Complete Global on another page')); - assert.ok(text.includes('Exact page LLM detection data')); + assert.ok(text.includes('LLM detection by page')); assert.ok(text.includes('Every page decision for Shared defect')); } const identities = await page.locator('dt').filter({hasText: /^Finding ID$/}).evaluateAll(elements => @@ -207,19 +225,17 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await global.locator('summary').first().focus(); await plain.keyboard.press('Enter'); assert.equal(await global.locator('details[open]').count(), 1); - if (filename === fullFile) { - for (const n of [0, 1]) { - await global.locator('details details > summary').nth(n).focus(); - await plain.keyboard.press('Enter'); - } - assert.equal(await global.locator('details[open]').count(), 3); + for (const n of filename === fullFile ? [0, 1] : [0]) { + await global.locator('details details > summary').nth(n).focus(); + await plain.keyboard.press('Enter'); } + assert.equal(await global.locator('details[open]').count(), filename === fullFile ? 3 : 2); if (behavior === 'print') { const pdf = path.join(path.dirname(filename), `${path.basename(filename)}-nojs.pdf`); await plain.pdf({path: pdf, format: 'A4'}); const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); for (const expected of ['Shared defect', 'Eligibility classification', 'Complete Global on home', - 'Detection overview', 'Exact detection overview data', 'Saved pricing version']) { + 'Detection overview', 'Estimated run cost:']) { assert.ok(text.includes(expected), `Basic no-JavaScript PDF includes ${expected}`); } } diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 98338b1..854a3e6 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -4,6 +4,7 @@ import hashlib import json from contextlib import nullcontext +from decimal import Decimal from pathlib import Path from types import SimpleNamespace @@ -326,14 +327,29 @@ def client(**kwargs): edit_review(run, *[{"classification": "llm_eligible"}] * 6) text = report(run) assert model in text and run.name in text - assert "published prices captured on 2026-10-09" in text - assert "18000 input tokens" in text and "3600 output tokens" in text - assert "9000 cached input tokens" in text and "1800 cache-creation input tokens" in text + assert f"Estimated run cost: ${Decimal(cost):.6f} USD" in text + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + usage = document.select_one('.token-usage') + assert [node.get_text() for node in usage.select('dd')] == ["18,000", "3,600", "9,000", "1,800"] + pricing = document.select_one('.model-pricing') + prices = load(run, "artifacts/pricing.json")["models"][model] + first_tier = pricing.select_one('tbody tr') + assert [cell.get_text() for cell in first_tier.select('td')] == [ + '$' + prices[key] for key in ( + 'input_per_million_usd', 'output_per_million_usd', + 'cached_input_per_million_usd', 'cache_creation_input_per_million_usd', + ) + ] + if 'long_context' in prices: + assert pricing.select('tbody tr')[1].th.get_text() == f"Over {prices['long_context']['above_input_tokens']:,}" -@pytest.mark.parametrize("model,cost", [("gpt-6-luna", "0.0036"), ("gpt-6.1-sol", "0.072")]) +@pytest.mark.parametrize("model,cost,display_name", [ + ("gpt-6-luna", "0.0036", "GPT-6 Luna"), + ("gpt-6.1-sol", "0.072", "GPT-6.1 Sol"), +]) def test_gpt_presets_use_openai_key_and_save_response_identity( - workspace, monkeypatch, model, cost, + workspace, monkeypatch, model, cost, display_name, ): """GPT requests use the selected preset and preserve provider evidence for review.""" requests, options = [], [] @@ -386,7 +402,10 @@ def client(**kwargs): assert findings and all(item["model"] == model for item in findings) edit_review(run, *[{"classification": "llm_eligible"}] * 6) text = report(run) - assert model in text and "18000 input tokens" in text and "3600 output tokens" in text + assert display_name in text and f"Estimated run cost: ${Decimal(cost):.6f} USD" in text + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert document.h1.get_text() == f"LLM Accessibility Evaluation: {display_name}" + assert display_name in document.select_one('.model-pricing').get_text() def test_run_help_explains_presets_and_provider_credentials(capsys): @@ -593,9 +612,13 @@ def test_historical_reports_keep_saved_prices_and_stored_estimate(workspace, mon run / "artifacts/raw-llm-responses.json")} text = report(run) assert "Estimated run cost: $5.40" in text - assert "published prices dated 2025-10-01" in text - assert "https://platform.claude.com/docs/en/about-claude/pricing" in text - assert "180000 input tokens" in text and "18000 output tokens" in text + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert document.select_one('#performance .performance-heading .cost-value').get_text(" ", strip=True) == "$5.400000 USD" + pricing = document.select_one('.model-pricing') + assert 'Published prices dated 2025-10-01' in pricing.get_text() + assert [cell.get_text() for cell in pricing.select('tbody tr td')] == ['$20', '$100'] + assert [node.get_text() for node in document.select('.token-usage dd')] == ['180,000', '18,000'] + assert not document.select('#provenance, a[href="#provenance"]') assert all(path.read_bytes() == saved for path, saved in originals.items()) @@ -607,7 +630,9 @@ def test_report_describes_the_evaluated_page(workspace, monkeypatch): run = only_run(workspace) edit_review(run, *[{"classification": "llm_eligible"}] * 6) text = report(run) - assert "This evaluation covers the saved page at https://example.test/contact/" in text + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert "https://example.test/contact/" in document.header.get_text() + assert document.h1.get_text() == f"LLM Accessibility Evaluation: {load(run, 'run.json')['configuration']['model']}" assert "homepage only" not in text @@ -656,22 +681,17 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): assert "cannot claim coverage" in refused(run, capsys) edit_review(run, *decisions) text = report(run) - assert ("Of the 2 Human findings the LLM could be expected to detect from the " - "page's HTML, it caught 1 (50.0%)") in text - assert "they caught 1 (50.0%)" in text - assert "Of all 6 Human findings, either part of the tool caught 3 (50.0%)" in text document = BeautifulSoup((run / "report.html").read_text(), "html.parser") chart = document.find("figure", attrs={"aria-labelledby": "detection-chart-title"}) assert chart is not None - assert "LLM: 1 caught / 2 eligible; 50.0%" in chart.get_text(" ", strip=True) - table = document.find("caption", string="Exact detection overview data").parent - assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] - for row in table.tbody.find_all("tr")] == [ - ["LLM", "1", "2", "50.0%", "Human findings expected to be detected by the LLM"], - ["Programmatic", "1", "2", "50.0%", "Human findings expected to be detected by Programmatic checks"], - ["Overall", "3", "6", "50.0%", "All imported Human findings, including findings Not testable from the page file"], + assert [item.get_text(" ", strip=True) for item in chart.select(".chart-value")] == [ + "LLM: 1 caught / 2 eligible; 50.0%", + "Programmatic: 1 caught / 2 eligible; 50.0%", + "Overall: 3 caught / 6 eligible; 50.0%", ] - assert document.find("caption", string="Exact page LLM detection data") is None + assert not document.select(".detection-chart table") + assert "Bar length shows" not in text + assert "LLM detection rate: Of" not in text rows = document.select("#findings tbody > tr") assert [cell.get_text(" ", strip=True) for cell in rows[2].find_all(["th", "td"], recursive=False)[:5]] == [ "Skipped heading", "1.3.1", "Programmatic checks", "Caught", "Not caught", @@ -680,16 +700,13 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): assert "No full finding" in rows[5].details.get_text() assert "Needs a browser" in rows[3].details.get_text() assert "Needs judgment" in rows[4].details.get_text() - assert ("1 of 18 LLM checks returned an Unreadable response that the tool " - "would skip: page title check.") in text + assert load(run, "run.json")["format_failures"] == ["page_title"] assert llm[0] in text and llm[1] in text and programmatic in text assert llm[2] not in text edit_review(run, *[{"classification": "ambiguous", "llm_finding_id": "", "programmatic_finding_id": "", "match_reason": ""}] * 6) text = report(run) - assert "No Human findings were expected to be detected by the LLM" in text - assert "No Human findings were expected to be detected by the Programmatic checks" in text - assert "Of all 6 Human findings, either part of the tool caught 0 (0.0%)" in text + assert "Overall: 0 caught / 6 eligible; 0.0%" in text @pytest.mark.parametrize("failure,calls,cost,reason", [ @@ -838,19 +855,27 @@ def test_single_page_report_is_standalone_html(workspace, monkeypatch, capsys): assert document.title and document.h1 assert document.select_one('a[href="#main"]') and document.main["id"] == "main" assert document.style and not document.select("script[src], link[rel=stylesheet], img[src]") + performance = document.select_one('#performance') + assert performance.find('div', recursive=False)['class'] == ['performance-heading'] + assert performance.select_one('.run-cost .cost-value') + assert not document.select('#provenance, a[href="#provenance"]') table = document.find("table", attrs={"id": "findings"}) assert table.caption and len(table.tbody.find_all("tr", recursive=False)) == 6 assert [row.th.get_text(strip=True) for row in table.tbody.find_all("tr", recursive=False)] == [ "Unclear link", "Missing destination", "Skipped heading", "Hidden menu", "Unclear claim", "No captions", ] disclosures = table.find_all("details") - assert len(disclosures) == 6 and all(not item.has_attr("open") for item in disclosures) - assert all("View evidence" in item.summary.get_text() for item in disclosures) + assert len(disclosures) == 12 and all(not item.has_attr("open") for item in disclosures) + page_panels = table.select('details[data-page-decision]') + assert len(page_panels) == 6 + assert all("View Page Decisions:" in item.summary.get_text() for item in page_panels) + assert all(item.find_parent('details') is not None for item in page_panels) assert "Vague link\nfor everyone" in document.get_text() -def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monkeypatch): - """Editable evidence cannot replace originals, and joint matches retain check provenance.""" +@pytest.mark.parametrize("match_count", [1, 2]) +def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monkeypatch, match_count): + """Single and joint matches retain originals, provenance and escaped evidence.""" fake_client(monkeypatch) assert run_cli(workspace, "--live", "--approve-live") == 0 run = only_run(workspace) @@ -859,7 +884,7 @@ def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monke findings[0].update(problem=hostile, element="", location="Footer", checklist="Synthetic checklist 42") (run / "normalized-llm-findings.json").write_text(json.dumps(findings)) edit_review(run, { - "classification": "llm_eligible", "llm_finding_id": ";".join(item["finding_id"] for item in findings[:2]), + "classification": "llm_eligible", "llm_finding_id": ";".join(item["finding_id"] for item in findings[:match_count]), "match_reason": "Together they fully cover the link", "review_notes": hostile, "actual result": "Editable evidence replacement", "Issue Title": "Edited title", }, *[{"classification": "ambiguous"}] * 5) @@ -872,23 +897,21 @@ def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monke assert "Vague link\nfor everyone" in text and "Unclear link" in text assert hostile in text and not document.select("header script, #findings script, img, [onerror]") assert "Synthetic checklist 42" in text - assert all(item["finding_id"] in text for item in findings[:2]) + assert all(item["finding_id"] in text for item in findings[:match_count]) assert "Together they fully cover the link" in text and "Footer" in text assert "" in text and findings[0]["page_url"] in text + matches = evidence.select('.matched') + assert len(matches) == match_count -@pytest.mark.parametrize("missing", ["human_audit_filename", "configuration.model"]) -def test_report_requires_complete_rendering_data(workspace, monkeypatch, capsys, missing): +def test_report_requires_complete_rendering_data(workspace, monkeypatch, capsys): """Absent required fields cannot publish HTML or discard a legacy report.""" fake_client(monkeypatch) assert run_cli(workspace, "--live", "--approve-live") == 0 run = only_run(workspace) edit_review(run, *[{"classification": "llm_eligible"}] * 6) manifest = load(run, "run.json") - if missing == "configuration.model": - del manifest["configuration"]["model"] - else: - del manifest[missing] + del manifest["configuration"]["model"] (run / "run.json").write_text(json.dumps(manifest)) legacy = run / "report.md" legacy.write_text("Previous report") @@ -911,12 +934,9 @@ def test_report_with_no_human_findings_has_no_false_zero_rates(workspace, monkey csv.writer(stream).writerow(header) text = report(run) assert "No Human findings were imported for this page" in text - assert "No Human findings were expected to be detected by the LLM" in text - assert "No Human findings were expected to be detected by the Programmatic checks" in text assert "(0.0%)" not in text document = BeautifulSoup((run / "report.html").read_text(), "html.parser") - table = document.find("caption", string="Exact detection overview data").parent - assert [row.find_all("td")[2].get_text(strip=True) for row in table.tbody.find_all("tr")] == [ + assert [item.get_text(strip=True) for item in document.select(".chart-percentage")] == [ "Not defined: no imported Human findings" if index == 2 else "Not defined: no eligible Human findings" for index in range(3) ] diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 26967b8..43fb138 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -123,32 +123,33 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert report_all(run) == 0 document = BeautifulSoup((run / "report.html").read_text(), "html.parser") text = document.get_text(" ", strip=True) - assert "LLM detection rate: Of the 4" in text and "caught 2 (50.0%)" in text - assert "Programmatic detection rate: Of the 3" in text and "caught 1 (33.3%)" in text - assert "Overall detection rate: Of all 8 Human findings" in text and "caught 5 (62.5%)" in text - overview = document.find("caption", string="Exact detection overview data").parent - assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])[:4]] - for row in overview.tbody.find_all("tr")] == [ - ["LLM", "2", "4", "50.0%"], - ["Programmatic", "1", "3", "33.3%"], - ["Overall", "5", "8", "62.5%"], + overview = document.find("figure", attrs={"aria-labelledby": "detection-chart-title"}) + assert [item.get_text(" ", strip=True) for item in overview.select(".chart-value")] == [ + "LLM: 2 caught / 4 eligible; 50.0%", + "Programmatic: 1 caught / 3 eligible; 33.3%", + "Overall: 5 caught / 8 eligible; 62.5%", ] - comparison = document.find("caption", string="Exact page LLM detection data") - assert comparison is not None - assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] - for row in comparison.parent.tbody.find_all("tr")] == [ - [URLS[0], "1", "3", "33.3%"], - [URLS[1], "1", "3", "33.3%"], - [URLS[2], "1", "2", "50.0%"], + comparison = document.find("figure", attrs={"aria-labelledby": "page-chart-title"}) + assert [item.get_text(" ", strip=True) for item in comparison.select(".chart-value")] == [ + f"{URLS[0]}: 1 caught / 3 eligible; 33.3%", + f"{URLS[1]}: 1 caught / 3 eligible; 33.3%", + f"{URLS[2]}: 1 caught / 2 eligible; 50.0%", ] + assert not document.select(".detection-chart table") + assert "Each original Human finding counts once." not in text assert "Repeated Global findings contribute to each page's results" in text assert "page rates must not be added or averaged" in text assert "not a controlled ranking of pages or model quality" in text assert "Review decisions by: Combined reviewer" in text - assert "Estimated run cost: $0.00" in text - assert "1000 input tokens" in text and "100 output tokens" in text - assert "published prices captured on 2026-10-09" in text - assert "Unreadable responses: 0" in text + assert document.select_one('#performance .performance-heading .cost-value').get_text(" ", strip=True) == "$0.000150 USD" + assert [node.get_text() for node in document.select('.token-usage dd')] == [ + f"{manifest['usage']['input_tokens']:,}", f"{manifest['usage']['output_tokens']:,}", + ] + assert len(document.select('.model-pricing table')) == 1 + tiers = document.select('.model-pricing tbody tr') + assert [cell.get_text() for cell in tiers[0].select('td')] == ['$0.10', '$0.50'] + assert [cell.get_text() for cell in tiers[1].select('td')] == ['$0.20', '$0.75'] + assert not document.select('#provenance, a[href="#provenance"]') originals = json.loads((run / "human-findings.json").read_text()) overview = document.select("#findings > tbody > tr") assert [row.find("th").get_text(strip=True) for row in overview] == [ @@ -175,8 +176,8 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert finding["problem"] in text assert finding["prompt"] in text assert finding["run_id"] in text - assert "complete detection on any single page" in text - assert "partial matches across pages" in text + assert "complete detection on any single page" not in text + assert "partial matches across pages" not in text assert not document.select("[open]") assert not document.select("script[src], link[rel=stylesheet], iframe") @@ -252,15 +253,12 @@ def test_empty_page_without_globals_still_has_a_report(full_workspace, monkeypat assert report_all(run) == 0 empty = run / manifest["pages"][2]["run_dir"] text = BeautifulSoup((empty / "report.html").read_text(), "html.parser").get_text(" ", strip=True) - assert "Overall detection rate: No Human findings were imported for this page" in text - assert "Overall detection rate: Of all 3 Human findings" in BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) + assert "Overall: 0 caught / 0 eligible; Not defined: no imported Human findings" in text parent = BeautifulSoup((run / "report.html").read_text(), "html.parser") - comparison = parent.find("caption", string="Exact page LLM detection data").parent - assert [[cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"])] - for row in comparison.tbody.find_all("tr")] == [ - [URLS[0], "0", "0", "Not defined: no eligible Human findings"], - [URLS[1], "0", "0", "Not defined: no eligible Human findings"], - [URLS[2], "0", "0", "Not defined: no eligible Human findings"], + assert "Overall: 0 caught / 3 eligible; 0.0%" in parent.get_text(" ", strip=True) + comparison = parent.find("figure", attrs={"aria-labelledby": "page-chart-title"}) + assert [item.get_text(" ", strip=True) for item in comparison.select(".chart-value")] == [ + f"{url}: 0 caught / 0 eligible; Not defined: no eligible Human findings" for url in URLS ] @@ -334,10 +332,9 @@ def test_no_eligible_sources_retain_all_originals_in_overall(completed_review): save_review(child, rows) assert report_all(run) == 0 text = BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) - assert "LLM detection rate: No Human findings were expected" in text - assert "Programmatic detection rate: No Human findings were expected" in text - assert "Overall detection rate: Of all 8 Human findings" in text - assert "caught 0 (0.0%)" in text + assert "LLM: 0 caught / 0 eligible; Not defined: no eligible Human findings" in text + assert "Programmatic: 0 caught / 0 eligible; Not defined: no eligible Human findings" in text + assert "Overall: 0 caught / 8 eligible; 0.0%" in text def test_completed_child_report_survives_later_parent_failure(full_workspace, monkeypatch, capsys): @@ -363,7 +360,7 @@ def test_reviewer_description_is_required(completed_review, capsys): assert "reviewer description" in refused_report(run, capsys, reviewer=" ") -def test_combined_report_summarizes_unreadable_responses(full_workspace, monkeypatch): +def test_combined_report_keeps_unreadable_response_evidence(full_workspace, monkeypatch): """Successful Unreadable responses remain complete and are counted across pages.""" monkeypatch.setenv("OPENAI_API_KEY", "synthetic") @@ -386,7 +383,11 @@ def create(**options): row.update(classification="llm_eligible", classification_reason="HTML evidence") save_review(child, rows) assert report_all(run) == 0 - assert "Unreadable responses: 6 of 6 LLM checks" in BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text(" ", strip=True) + assert sum(len(json.loads((child / "run.json").read_text())["format_failures"]) + for child, _ in children(run, manifest)) == 6 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert not document.select('#provenance') + assert document.select_one('#performance .cost-value') def test_report_all_help_explains_review_completion(capsys): @@ -400,7 +401,7 @@ def test_report_all_help_explains_review_completion(capsys): def test_global_evidence_includes_every_page_without_deeper_disclosures(completed_review): - """Global evidence exposes misses and not-testable decisions in two levels.""" + """Global and page-local evidence use two disclosure levels.""" run, manifest = completed_review assert report_all(run) == 0 document = BeautifulSoup((run / "report.html").read_text(), "html.parser") @@ -413,7 +414,7 @@ def test_global_evidence_includes_every_page_without_deeper_disclosures(complete assert len(page_panels) == len(manifest["pages"]) assert all(not panel.find("details") for panel in page_panels) assert all(not panel.has_attr("name") for panel in page_panels) - assert all("View page evidence:" in panel.summary.get_text() for panel in page_panels) + assert all("View Page Decisions:" in panel.summary.get_text() for panel in page_panels) compact = outer.find("table") assert len(compact.select("tbody > tr")) == len(manifest["pages"]) assert [tr.find("th").get_text(strip=True) for tr in compact.select("tbody > tr")] == [ @@ -422,7 +423,13 @@ def test_global_evidence_includes_every_page_without_deeper_disclosures(complete assert "Not testable from the page file" in not_testable assert "Not caught" in not_testable local = next(row for row in rows if row.find("th").get_text(strip=True) == "First local") - assert len(local.find_all("details")) == 1 + assert len(local.find_all("details")) == 2 + panel = local.select_one('details[data-page-decision]') + assert panel.find_parent('details') is local.find('details') + assert panel['data-page'] == URLS[0] + assert f"View Page Decisions: {URLS[0]}" in panel.summary.get_text() + assert "Complete first issue" in panel.get_text() + assert not document.select('details[open]') def test_successful_full_regeneration_replaces_only_its_generated_markdown(completed_review, capsys): @@ -441,12 +448,12 @@ def test_successful_full_regeneration_replaces_only_its_generated_markdown(compl assert (historical / "report.md").read_text() == "historical result" -def test_missing_parent_rendering_field_withholds_parent_and_preserves_children(completed_review, capsys): - """Required provenance cannot silently disappear from a published parent.""" +def test_unused_parent_display_metadata_is_optional(completed_review): + """Reports do not require metadata from the removed provenance section.""" run, manifest = completed_review assert report_all(run) == 0 del manifest["human_audit_filename"] save_json(run / "full.json", manifest) - assert "Cannot render complete report" in refused_report(run, capsys) + assert report_all(run) == 0 assert all((run / page["run_dir"] / "report.html").exists() for page in manifest["pages"]) assert not list(run.glob(".report.html.*.tmp")) diff --git a/vision_aid/evaluation/css/charts.css b/vision_aid/evaluation/css/charts.css index f4107f2..d401ae5 100644 --- a/vision_aid/evaluation/css/charts.css +++ b/vision_aid/evaluation/css/charts.css @@ -1,9 +1,26 @@ +.performance-heading { margin-bottom: 1.5rem; } +.performance-heading h2 { margin: 0 0 1rem; } +.cost-overview { display: grid; grid-template-columns: minmax(0, .8fr) minmax(0, 1.2fr); gap: 1.5rem; align-items: start; } +.cost-overview > * { min-width: 0; } +@media (min-width: 60rem) { .cost-overview { align-items: stretch; } } +.run-cost { margin: 0; padding: .75rem 1rem; max-width: 100%; border: 3px solid #164e88; background: #edf2f6; } +.cost-label { display: block; margin-bottom: .4rem; font-weight: 650; } +.cost-value { display: block; font-size: clamp(2rem, 4vw, 3rem); line-height: 1.1; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.cost-currency { font-size: 1rem; font-weight: 650; } +.token-usage { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .4rem 1rem; margin: 1.25rem 0 0; padding-top: .75rem; border-top: 1px solid #687f93; } +.token-usage dd { margin: 0; text-align: right; font-variant-numeric: tabular-nums; } +.model-pricing h3 { margin: 0; } +.model-pricing table { min-width: 26rem; margin-bottom: 0; } +.model-pricing th { overflow-wrap: normal; } +.model-pricing td { white-space: nowrap; } +@media (max-width: 60rem) { .cost-overview { grid-template-columns: minmax(0, 1fr); } } .detection-chart { margin: 1rem 0; } .chart-bars { list-style: none; padding: 0; } .chart-bars li { margin-block: 1rem; padding: .75rem; border: 1px solid #687f93; overflow-wrap: anywhere; } -.chart-bars .chart-primary { border: 3px solid #164e88; } +.chart-bars .chart-primary { border: 3px solid #164e88; background: #edf2f6; } .chart-primary .chart-value { font-size: 1.2rem; } .chart-value { margin: 0 0 .5rem; } +.chart-percentage { text-decoration: underline; text-underline-offset: .15em; } .chart-track { width: 100%; height: 1.4rem; background: #edf2f6; border: 1px solid #687f93; } .chart-fill { display: block; height: 100%; background: #164e88; } @media (forced-colors: active) { .chart-fill { background: Highlight; } } diff --git a/vision_aid/evaluation/html_report.py b/vision_aid/evaluation/html_report.py index 922e8c0..90e6f3d 100644 --- a/vision_aid/evaluation/html_report.py +++ b/vision_aid/evaluation/html_report.py @@ -17,10 +17,11 @@ select_autoescape, ) -from vision_aid.evaluation.review import _rate, _usage_description, detection_counts +from vision_aid.evaluation.review import detection_counts if TYPE_CHECKING: from vision_aid.evaluation.page import PageReview + from vision_aid.evaluation.pricing import PriceSchedule EXPECTED = { "llm_eligible": "LLM", @@ -30,22 +31,67 @@ } -def _chart_rates(counts: dict[str, dict[str, int]], *, full: bool = False) -> list[dict[str, Any]]: +def _display_model(model: str) -> str: + """Use uppercase GPT in report labels while preserving saved model IDs.""" + return "GPT-" + model[4:].replace("-", " ").title() if model.startswith("gpt-") else model + + +def _cost_details(manifest: dict[str, Any], schedules: list[PriceSchedule]) -> dict[str, Any]: + """Present saved usage and model prices without recalculating the run cost.""" + usage = manifest["usage"] + model = manifest["configuration"].get("model") + if model is None: + raise ValueError("Cannot render complete report: model is missing") + token_usage = [ + {"label": label, "count": f"{usage[key]:,}"} + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("cached_input_tokens", "Cached input tokens"), + ("cache_creation_input_tokens", "Cache-write input tokens"), + ) + if key in {"input_tokens", "output_tokens"} or usage[key] + ] + model_prices = [] + seen = set() + for schedule in schedules: + if schedule.identity in seen: + continue + seen.add(schedule.identity) + rates = schedule.data["models"][model] + upper = rates.get("long_context") + tiers = [{"label": "All requests", "rates": rates}] + if upper: + threshold = f"{upper['above_input_tokens']:,}" + tiers = [ + {"label": f"Up to {threshold}", "rates": rates}, + {"label": f"Over {threshold}", "rates": upper}, + ] + model_prices.append({ + "model": _display_model(model), "description": schedule.price_description, + "tiers": tiers, + "show_cache_read": bool(usage["cached_input_tokens"]), + "show_cache_write": bool(usage["cache_creation_input_tokens"]), + }) + return { + "cost": f"{Decimal(manifest['estimated_cost_usd']):.6f}", + "model_name": _display_model(model), + "token_usage": token_usage, "model_prices": model_prices, + } + + +def _chart_rates(counts: dict[str, dict[str, int]]) -> list[dict[str, Any]]: """Prepare chart labels and widths from the existing deterministic counts.""" rates = [] - for kind, label, denominator in ( - ("llm", "LLM", "Human findings expected to be detected by the LLM"), - ("programmatic", "Programmatic checks", "Human findings expected to be detected by Programmatic checks"), - ("overall", "Overall", "All imported Human findings, including findings Not testable from the page file"), + for kind, label in ( + ("llm", "LLM"), + ("programmatic", "Programmatic"), + ("overall", "Overall"), ): caught, eligible = counts[kind]["caught"], counts[kind]["eligible"] percentage = f"{caught / eligible * 100:.1f}" if eligible else None - description = _rate(label, caught, eligible) - if full: - description = description.replace("the page's HTML", "the evaluated pages' HTML") rates.append({ - "label": label.removesuffix(" checks"), **counts[kind], - "description": description, "denominator": denominator, + "label": label, **counts[kind], "bar_percent": percentage, "percentage": percentage + "%" if percentage is not None else ( "Not defined: no imported Human findings" if kind == "overall" @@ -79,8 +125,6 @@ def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: "check_id": finding["prompt"], "checklist": finding["checklist"], "problem": finding["problem"], - "page_url": finding["page_url"], - "run_id": finding["run_id"], "element": finding["element"], "location": finding["location"], } @@ -89,6 +133,8 @@ def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: { "human": human, "decision": decision, + "source_url": manifest["source_url"], + "run_id": manifest["run_id"], "expected": EXPECTED[classification], "llm_outcome": "Caught" if row["matches"]["llm"] else "Not caught", "programmatic_outcome": "Caught" @@ -99,27 +145,12 @@ def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: ) rates = _chart_rates(counts) return { - "title": "LLM accessibility evaluation", + "title": "LLM Accessibility Evaluation", "manifest": manifest, "reviewer": reviewer, "rows": rows, "rates": rates, - "cost": f"{Decimal(manifest['estimated_cost_usd']):.2f}", - "pricing_description": review.schedule.price_description, - "pricing_identity": review.schedule.identity, - "pricing_version": review.schedule.version, - "pricing_sources": review.schedule.data.get( - "sources", - [review.schedule.data["source"]] - if "source" in review.schedule.data - else [], - ), - "usage_description": _usage_description(manifest["usage"]), - "unreadable": len(manifest["format_failures"]), - "prompts": len(manifest["prompts"]), - "failed_checks": ", ".join( - name.replace("_", " ") + " check" for name in manifest["format_failures"] - ), + **_cost_details(manifest, [review.schedule]), } @@ -147,20 +178,10 @@ def full_data( "pages": grouped[row["human_finding"]["human_finding_id"]], }) return { - "title": "Full accessibility evaluation", "full": True, + "title": "Full Accessibility Evaluation", "full": True, "manifest": manifest, "reviewer": reviewer, "rows": rows, "pages": pages, - "rates": _chart_rates(counts, full=True), - "cost": f"{Decimal(manifest['estimated_cost_usd']):.2f}", - "pricing_description": ", ".join(dict.fromkeys(page["pricing_description"] for page in pages)), - "pricing_schedules": list({ - page["pricing_identity"]: { - "version": page["pricing_version"], "identity": page["pricing_identity"], - "sources": page["pricing_sources"], - } for page in pages - }.values()), - "usage_description": _usage_description(manifest["usage"]), - "unreadable": sum(page["unreadable"] for page in pages), - "prompts": sum(page["prompts"] for page in pages), + "rates": _chart_rates(counts), + **_cost_details(manifest, [review.schedule for review in reviews]), } diff --git a/vision_aid/evaluation/templates/base.html.jinja b/vision_aid/evaluation/templates/base.html.jinja index 69dbd4d..8e7190e 100644 --- a/vision_aid/evaluation/templates/base.html.jinja +++ b/vision_aid/evaluation/templates/base.html.jinja @@ -9,12 +9,12 @@
-

{{ title }}

-

{% block scope %}{% endblock %}

-

Model: {{ manifest.configuration.model }}. Run ID: {{ manifest.run_id }}.

+

{{ title }}: {{ model_name }}

+{% block scope %}{% endblock %} +

Run ID: {{ manifest.run_id }}.

Review decisions by: {{ reviewer }}

- +
{% block content %}{% endblock %} diff --git a/vision_aid/evaluation/templates/charts.html.jinja b/vision_aid/evaluation/templates/charts.html.jinja index dffc400..1ce4ece 100644 --- a/vision_aid/evaluation/templates/charts.html.jinja +++ b/vision_aid/evaluation/templates/charts.html.jinja @@ -1,20 +1,35 @@ +{% macro performance_header(cost, token_usage, model_prices) %} +
+

Cost and detection results

+
+
Estimated run cost:${{ cost }} USD +
{% for item in token_usage %}
{{ item.label }}
{{ item.count }}
{% endfor %}
+
+

Listed model prices

+{% for prices in model_prices %} +

{{ prices.model }}. {{ prices.description | capitalize }}.

+
+ +{% if prices.show_cache_read %}{% endif %}{% if prices.show_cache_write %}{% endif %} +{% for tier in prices.tiers %}{% if prices.show_cache_read %}{% endif %}{% if prices.show_cache_write %}{% endif %}{% endfor %} +
USD per 1 million tokens
Input tokens per requestInputOutputCache readCache write
{{ tier.label }}${{ tier.rates.input_per_million_usd }}${{ tier.rates.output_per_million_usd }}${{ tier.rates.cached_input_per_million_usd }}${{ tier.rates.cache_creation_input_per_million_usd }}
+{% endfor %} +
+
+
+{% endmacro %} + {% macro rate_bar(label, rate, primary=false) %} -

{{ label }}: {{ rate.caught }} caught / {{ rate.eligible }} eligible; {{ rate.percentage }}

+

{{ label }}: {{ rate.caught }} caught / {{ rate.eligible }} eligible; {{ rate.percentage }}

{% if rate.bar_percent is not none %}{% endif %} {% endmacro %} {% macro detection_overview(rates) %}
-

Detection overview

-

Bar length shows the detection rate on a scale from 0% to 100%. The LLM result leads; each source uses its own denominator. No bar is drawn when the rate is not defined.

+

Detection overview

    {% for rate in rates %}{{ rate_bar(rate.label, rate, loop.first) }}{% endfor %}
-
- - -{% for rate in rates %}{% endfor %} -
Exact detection overview data
Detection sourceCaughtEligibleDetection rateDenominator
{{ rate.label }}{{ rate.caught }}{{ rate.eligible }}{{ rate.percentage }}{{ rate.denominator }}
{% endmacro %} @@ -22,12 +37,7 @@

LLM detection by page

Pages appear in Human audit order. Each bar uses a 0% to 100% scale and the page's LLM-eligible findings. Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged. This comparison is not a controlled ranking of pages or model quality.

-

No bar is drawn where no Human findings were eligible for the LLM. Exact values follow in the table.

+

No bar is drawn where no Human findings were eligible for the LLM.

    {% for page in pages %}{{ rate_bar(page.manifest.source_url, page.rates[0]) }}{% endfor %}
-
- - -{% for page in pages %}{% endfor %} -
Exact page LLM detection data
PageLLM caughtLLM eligibleLLM detection rate
{{ page.manifest.source_url }}{{ page.rates[0].caught }}{{ page.rates[0].eligible }}{{ page.rates[0].percentage }}
{% endmacro %} diff --git a/vision_aid/evaluation/templates/evidence.html.jinja b/vision_aid/evaluation/templates/evidence.html.jinja index 5cb17f7..2fbf934 100644 --- a/vision_aid/evaluation/templates/evidence.html.jinja +++ b/vision_aid/evaluation/templates/evidence.html.jinja @@ -4,7 +4,13 @@ {% for label, value in row.human.raw_evidence.items() %}
{{ label }}
{{ value }}
{% endfor %} {% endmacro %} {% macro decision_evidence(row) %} -

Review decision

-
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
Review notes
{{ row.decision.review_notes }}
-{% for finding in row.matched %}

{{ finding.source }} matching evidence

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
Page provenance
{{ finding.page_url }}; run {{ finding.run_id }}
Element
{{ finding.element }}
Location
{{ finding.location }}
{% endfor %} +

{{ row.source_url }}

+

Run ID: {{ row.run_id }}

+
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
VA-DAT Matched Findings
{{ row.decision.review_notes }}
+{% for finding in row.matched %}

{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}
Element
{{ finding.element }}
{% endif %}
Location
{{ finding.location }}
{% endfor %} +{% endmacro %} +{% macro page_evidence(row) %} +
View Page Decisions: {{ row.source_url }} +
{{ decision_evidence(row) }}
+
{% endmacro %} diff --git a/vision_aid/evaluation/templates/full.html.jinja b/vision_aid/evaluation/templates/full.html.jinja index cbbef06..4e5c51e 100644 --- a/vision_aid/evaluation/templates/full.html.jinja +++ b/vision_aid/evaluation/templates/full.html.jinja @@ -1,31 +1,27 @@ {% extends 'base.html.jinja' %} -{% from 'evidence.html.jinja' import human_evidence, decision_evidence %} -{% from 'charts.html.jinja' import detection_overview, page_comparison %} +{% from 'evidence.html.jinja' import human_evidence, page_evidence %} +{% from 'charts.html.jinja' import performance_header, detection_overview, page_comparison %} -{% block document_title %}{{ title }}: {{ manifest.run_id }}{% endblock %} -{% block scope %}This Full evaluation covers the saved HTML for every page listed in the Human audit with one fixed model.{% endblock %} +{% block document_title %}{{ title }}: {{ model_name }}; {{ manifest.run_id }}{% endblock %} {% block content %}
-

Detection results

+{{ performance_header(cost, token_usage, model_prices) }} {{ detection_overview(rates) }} -{% for rate in rates %}

{{ rate.description }}

{% endfor %} -

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each original Human finding once.

-

Each original Human finding counts once. Page-local findings use their assigned page. Global findings earn credit through complete detection on any single page, independently for each detection source. Several catches still count once; partial matches across pages are never combined.

{{ page_comparison(pages) }}

Human findings

-

Each original Human finding appears once in source order. Evidence starts collapsed. Open View evidence for the original Human finding and its review decisions. Global findings include every page decision, with independently expandable page evidence.

+

Each original Human finding appears once in source order. Open View details to read the original Human finding, then View Page Decisions to read the review decision and matched tool findings. Global findings include a separate page decisions dropdown for every page. Details and page decisions start collapsed.

{% include 'controls.html.jinja' %}
- + {% for row in rows %} - {% endfor %}
Original Human findings and combined detection outcomes
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksEvidence
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksDetails
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }}
+
View details
{{ human_evidence(row) }}
{% if row.human.page_scope == 'global' %}

Combined result

@@ -34,13 +30,8 @@ {% for page in row.pages %}{% endfor %}
Every page decision for {{ row.human.problem }}
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
-{% for page in row.pages %}
View page evidence: {{ page.source_url }} for {{ row.human.problem }} -

Page: {{ page.source_url }}; run {{ page.run_id }}. LLM: {{ page.llm_outcome }}. Programmatic checks: {{ page.programmatic_outcome }}.

-
{{ decision_evidence(page) }}
-
{% endfor %} -{% else %} -{% for page in row.pages %}

Assigned page: {{ page.source_url }}; run {{ page.run_id }}.

{{ decision_evidence(page) }}
{% endfor %} {% endif %} +{% for page in row.pages %}{{ page_evidence(page) }}{% endfor %}
@@ -50,13 +41,4 @@

Eligibility is established independently of detection. If any page is testable for a Global finding, the finding enters that expected source's denominator once. All testable pages must agree on the source. Findings not testable on any page remain in Overall and outside the LLM and Programmatic denominators.

Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged.

-

Cost and provenance

-

Estimated run cost: ${{ cost }}, calculated using authoritative saved page totals, provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

-

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

-
Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
-{% for schedule in pricing_schedules %}
Saved pricing version
{{ schedule.version }}
Saved pricing SHA-256
{{ schedule.identity }}
{% for source in schedule.sources %}
Pricing source address
{{ source }}
{% endfor %}{% endfor %} -
-

Evaluated pages in source order

-
{% for page in pages %}
{{ page.manifest.source_url }}
Run: {{ page.manifest.run_id }}; Benchmark snapshot SHA-256: {{ page.manifest.snapshot_sha256 }}. Unreadable responses: {{ page.unreadable }} of {{ page.prompts }} LLM checks{% if page.unreadable %}; {{ page.failed_checks }}{% endif %}.
{% endfor %}
-
{% endblock %} diff --git a/vision_aid/evaluation/templates/page.html.jinja b/vision_aid/evaluation/templates/page.html.jinja index 072af46..0a9e4ae 100644 --- a/vision_aid/evaluation/templates/page.html.jinja +++ b/vision_aid/evaluation/templates/page.html.jinja @@ -1,32 +1,30 @@ {% extends 'base.html.jinja' %} -{% from 'evidence.html.jinja' import human_evidence, decision_evidence %} -{% from 'charts.html.jinja' import detection_overview, page_comparison %} +{% from 'evidence.html.jinja' import human_evidence, page_evidence %} +{% from 'charts.html.jinja' import performance_header, detection_overview, page_comparison %} -{% block document_title %}{{ title }}: {{ manifest.source_url }}{% endblock %} -{% block scope %}This evaluation covers the saved page at {{ manifest.source_url }}, including Global findings that apply there.{% endblock %} +{% block document_title %}{{ title }}: {{ model_name }}; {{ manifest.source_url }}{% endblock %} +{% block scope %}

{{ manifest.source_url }}

{% endblock %} {% block content %}
-

Detection results

+{{ performance_header(cost, token_usage, model_prices) }} {{ detection_overview(rates) }} -{% for rate in rates %}

{{ rate.description }}

{% endfor %} -

The LLM and Programmatic detection rates count only catches by the expected part. A catch by the other part counts toward the Overall detection rate, which counts each Human finding once out of all imported Human findings.

Human findings

-

Evidence starts collapsed. Open any View evidence control to read the original Human finding and the review decision.

+

Open View details to read the original Human finding, then View Page Decisions to read the review decision and matched tool findings. Details and page decisions start collapsed.

{% include 'controls.html.jinja' %}
- + {% for row in rows %} - {% endfor %} @@ -34,10 +32,4 @@ {% if not rows %}

No Human findings were imported for this page.

{% endif %} {% include 'method.html.jinja' %} -

Cost and provenance

-

Estimated run cost: ${{ cost }}, calculated using provider-reported token usage and the {{ pricing_description }}. Reported totals: {{ usage_description }}. The actual bill may differ slightly.

-

Unreadable responses: {{ unreadable }} of {{ prompts }} LLM checks. These responses yield no LLM findings and do not change execution completion.

-{% if unreadable %}

{{ unreadable }} of {{ prompts }} LLM checks returned an Unreadable response that the tool would skip: {{ failed_checks }}.

{% endif %} -
Human audit
{{ manifest.human_audit_filename }}
Human audit SHA-256
{{ manifest.human_audit_sha256 }}
Benchmark snapshot SHA-256
{{ manifest.snapshot_sha256 }}
Saved pricing version
{{ pricing_version }}
Saved pricing SHA-256
{{ pricing_identity }}
{% for source in pricing_sources %}
Pricing source address
{{ source }}
{% endfor %}
-
{% endblock %} From b77bd94909cbbfdca173bcadf6bb80a09145e2ab Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 16:21:11 -0400 Subject: [PATCH 096/126] feat(reports): simplified wording for llm detection rates for each page in the full report. --- vision_aid/evaluation/templates/charts.html.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vision_aid/evaluation/templates/charts.html.jinja b/vision_aid/evaluation/templates/charts.html.jinja index 1ce4ece..4ae7e2c 100644 --- a/vision_aid/evaluation/templates/charts.html.jinja +++ b/vision_aid/evaluation/templates/charts.html.jinja @@ -36,8 +36,8 @@ {% macro page_comparison(pages) %}

LLM detection by page

-

Pages appear in Human audit order. Each bar uses a 0% to 100% scale and the page's LLM-eligible findings. Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged. This comparison is not a controlled ranking of pages or model quality.

-

No bar is drawn where no Human findings were eligible for the LLM.

+

Repeated Global findings contribute to each page's results but only once to combined results.

+
    {% for page in pages %}{{ rate_bar(page.manifest.source_url, page.rates[0]) }}{% endfor %}
{% endmacro %} From 7de54ab84b92b756fd9c5297ec3569f667020ef4 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 16:32:54 -0400 Subject: [PATCH 097/126] feat(reports): simplify filters and explain overall counts Remove evidence search and its supporting markup, styles, and JavaScript while preserving page and outcome filters. Explain why Overall includes more Human findings than the source-eligible counts, and update browser checks for filtering and printing. --- tests/report_interactions.cjs | 50 +++---------------- tests/test_full_report.py | 3 +- tests/test_report_interactions.py | 2 +- vision_aid/evaluation/css/print.css | 2 +- vision_aid/evaluation/css/report.css | 5 +- vision_aid/evaluation/js/filters.js | 36 ++----------- .../evaluation/templates/charts.html.jinja | 1 + .../evaluation/templates/controls.html.jinja | 6 +-- .../evaluation/templates/evidence.html.jinja | 2 +- .../evaluation/templates/full.html.jinja | 4 +- .../evaluation/templates/page.html.jinja | 4 +- 11 files changed, 23 insertions(+), 92 deletions(-) diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs index 0e3559c..1a9bbca 100644 --- a/tests/report_interactions.cjs +++ b/tests/report_interactions.cjs @@ -18,6 +18,9 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); page.on('pageerror', error => errors.push(error.message)); await page.goto(pathToFileURL(filename).href); const rows = page.locator('#findings > tbody > tr'); + assert.equal(await page.locator('#finding-controls').isVisible(), true); + assert.equal(await page.locator('input[type=search], #finding-search, [data-search-matches]').count(), 0); + assert.equal(await page.locator('#finding-controls legend').textContent(), 'Filter findings'); const cost = page.locator('#performance .run-cost'); assert.equal(await cost.isVisible(), true); assert.match(await cost.textContent(), /Estimated run cost:\$0\.\d{6} USD/); @@ -29,38 +32,6 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= 320)); await page.setViewportSize({width: 1280, height: 900}); const visibleTitles = () => page.locator('#findings > tbody > tr:visible > th').allTextContents(); - if (behavior === 'search') { - assert.equal(await rows.locator('details[open]').count(), 0); - const search = page.getByLabel('Search findings and evidence'); - await search.fill('Complete Global on home'); - assert.deepEqual(await visibleTitles(), ['Shared defect']); - assert.equal(await search.evaluate(e => document.activeElement === e), true); - const match = page.getByRole('button', {name: /Show match in.*Complete Global on home/}); - assert.equal(await match.count(), 1); - assert.match(await match.textContent(), /https:\/\/example.test\//); - await match.click(); - const global = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); - assert.equal(await global.locator('details[open]').count(), 2); - if (filename === fullFile) { - await search.fill('Complete Global on another page'); - const pageMatches = page.getByRole('button', {name: /Show match in.*Complete Global on another page/}); - assert.equal(await pageMatches.count(), 2); - await pageMatches.nth(0).click(); - await pageMatches.nth(1).click(); - assert.equal(await global.locator('details details[open]').count(), 3); - } - await search.fill('Synthetic failure'); - assert.deepEqual(await visibleTitles(), filename === fullFile ? ['First local', 'Second local', 'Shared defect'] : ['First local', 'Second local']); - assert.ok(await page.getByRole('button', {name: /Show match in.*Synthetic failure/}).count()); - await search.fill('Original evidence'); - assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 5 : 4); - assert.ok(await page.getByRole('button', {name: /Show match in Original Human finding/}).count()); - await search.fill('does-not-appear-in-any-evidence'); - assert.equal(await rows.filter({visible: true}).count(), 0); - assert.match(await page.getByRole('status').textContent(), /No matching findings/); - await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); - assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 8 : 7); - } if (behavior === 'filters') { const performance = await page.locator('#performance').textContent(); const outcome = page.getByLabel('Detection outcome', {exact: true}); @@ -85,11 +56,6 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); const selected = shared.getByText(/Selected page:/); assert.match(await selected.textContent(), /https:\/\/example.test\/.*LLM: Not caught.*Programmatic checks: Caught/); assert.match(await shared.locator(':scope > td').nth(3).textContent(), /Caught/); - await page.getByLabel('Search findings and evidence').fill('Complete Global on another page'); - assert.deepEqual(await visibleTitles(), []); - await page.getByLabel('Search findings and evidence').fill('Complete Global on home'); - assert.deepEqual(await visibleTitles(), ['Shared defect']); - await page.getByLabel('Search findings and evidence').fill(''); await pageFilter.selectOption({label: 'https://example.test/contact/'}); assert.deepEqual(await visibleTitles(), ['Contact local', 'Partial across pages']); await outcome.selectOption({label: 'Programmatic missed (Programmatic eligible)'}); @@ -105,8 +71,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.equal(await page.getByRole('button', {name: 'Clear filters', exact: true}).evaluate(e => document.activeElement === e), true); } if (behavior === 'disclosures') { - const search = page.getByLabel('Search findings and evidence'); - await search.fill('Shared defect'); + await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'Programmatic caught (any eligibility)'}); if (filename === fullFile) await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); const expand = page.getByRole('button', {name: 'Expand all visible evidence', exact: true}); await expand.focus(); @@ -150,10 +115,9 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); } await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'LLM missed (LLM eligible)'}); - await page.getByLabel('Search findings and evidence').fill('Complete Global on home'); // Ordinary printing follows the current reading state. await page.emulateMedia({media: 'print'}); - assert.equal(await rows.filter({visible: true}).count(), 1); + assert.equal(await rows.filter({visible: true}).count(), 2); await page.emulateMedia({media: 'screen'}); const originalEvidence = await page.locator('main').textContent(); const savedState = () => page.evaluate(() => ({ @@ -177,7 +141,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await page.emulateMedia({media: 'print'}); assert.equal(await print.isVisible(), false); assert.equal(await page.locator('#finding-controls').isVisible(), false); - assert.equal(await page.locator('[data-search-matches]:visible, [data-selected-page-result]:visible').count(), 0); + assert.equal(await page.locator('[data-selected-page-result]:visible').count(), 0); const evidenceWidth = await rows.nth(0).locator('.evidence').evaluate(e => e.getBoundingClientRect().width); assert.ok(evidenceWidth > 600, `Print evidence uses the page width: ${evidenceWidth}`); const captionWidth = await page.locator('#findings > caption').evaluate(e => e.getBoundingClientRect().width); @@ -218,7 +182,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); const nojs = await browser.newContext({offline: true, javaScriptEnabled: false}); const plain = await nojs.newPage(); await plain.goto(pathToFileURL(filename).href); - assert.equal(await plain.getByLabel('Search findings and evidence').isVisible(), false); + assert.equal(await plain.locator('#finding-controls').isVisible(), false); assert.equal(await plain.getByRole('button', {name: 'Print complete report', exact: true}).isVisible(), false); assert.equal(await plain.locator('#findings > tbody > tr:visible').count(), filename === fullFile ? 8 : 7); const global = plain.locator('#findings > tbody > tr').filter({has: plain.getByRole('rowheader', {name: 'Shared defect', exact: true})}); diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 43fb138..9b24e67 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -139,7 +139,8 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert "Each original Human finding counts once." not in text assert "Repeated Global findings contribute to each page's results" in text assert "page rates must not be added or averaged" in text - assert "not a controlled ranking of pages or model quality" in text + assert "those the tool cannot detect from the saved HTML and those too ambiguous to evaluate" in overview.get_text() + assert "the Overall count can exceed their combined total" in overview.get_text() assert "Review decisions by: Combined reviewer" in text assert document.select_one('#performance .performance-heading .cost-value').get_text(" ", strip=True) == "$0.000150 USD" assert [node.get_text() for node in document.select('.token-usage dd')] == [ diff --git a/tests/test_report_interactions.py b/tests/test_report_interactions.py index d3770ad..d21fc88 100644 --- a/tests/test_report_interactions.py +++ b/tests/test_report_interactions.py @@ -22,7 +22,7 @@ __all__ = ["completed_review", "full_workspace"] -@pytest.mark.parametrize("behavior", ["search", "filters", "disclosures", "print"]) +@pytest.mark.parametrize("behavior", ["filters", "disclosures", "print"]) def test_offline_report_interactions(completed_review, tmp_path, behavior): """Users can investigate saved evidence without resources beside the HTML.""" modules = os.environ.get("REPORT_BROWSER_MODULES") diff --git a/vision_aid/evaluation/css/print.css b/vision_aid/evaluation/css/print.css index 5fcac7d..c42665c 100644 --- a/vision_aid/evaluation/css/print.css +++ b/vision_aid/evaluation/css/print.css @@ -3,7 +3,7 @@ @media print { body { max-width: none; padding: 0; color: #000; font-size: 11pt; line-height: 1.4; } .skip, nav, #print-complete-report, #finding-controls, - [data-search-matches], [data-selected-page-result] { display: none !important; } + [data-selected-page-result] { display: none !important; } .table-scroll { overflow: visible; } table { table-layout: fixed; } thead { display: table-header-group; } diff --git a/vision_aid/evaluation/css/report.css b/vision_aid/evaluation/css/report.css index ea967e9..4a5f9e7 100644 --- a/vision_aid/evaluation/css/report.css +++ b/vision_aid/evaluation/css/report.css @@ -26,7 +26,6 @@ dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } #finding-controls fieldset { border: 1px solid #8a99a6; padding: 1rem; } #finding-controls label { display: block; font-weight: 650; margin-top: .8rem; } -#finding-controls input, #finding-controls select { display: block; width: 100%; max-width: 42rem; font: inherit; padding: .5rem; color: inherit; background: #fff; border: 1px solid #687f93; } -#finding-controls button, [data-search-matches] button { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } -[data-search-matches] button { text-align: left; overflow-wrap: anywhere; max-width: 100%; } +#finding-controls select { display: block; width: 100%; max-width: 42rem; font: inherit; padding: .5rem; color: inherit; background: #fff; border: 1px solid #687f93; } +#finding-controls button { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } [data-selected-page-result] { border-left: 3px solid #687f93; padding-left: .7rem; } diff --git a/vision_aid/evaluation/js/filters.js b/vision_aid/evaluation/js/filters.js index fc7dc31..056f4cd 100644 --- a/vision_aid/evaluation/js/filters.js +++ b/vision_aid/evaluation/js/filters.js @@ -2,21 +2,12 @@ (() => { 'use strict'; const controls = document.getElementById('finding-controls'); - const search = document.getElementById('finding-search'); const pageFilter = document.getElementById('finding-page'); const outcome = document.getElementById('finding-outcome'); const status = document.getElementById('finding-count'); - const normalize = text => text.replace(/\s+/g, ' ').trim().toLocaleLowerCase(); const rows = [...document.querySelectorAll('#findings > tbody > tr')].map(row => { - const locations = [...row.querySelectorAll('[data-evidence-location]')].map(element => ({ - element, text: element.textContent.replace(/\s+/g, ' ').trim(), - label: element.dataset.evidenceLocation, - page: element.closest('[data-page-decision]')?.dataset.page, - })); - const overview = [...row.children].slice(0, -1).map(cell => cell.textContent).join(' '); - return {row, locations, decisions: [...row.querySelectorAll('[data-page-decision]')], - selected: row.querySelector('[data-selected-page-result]'), - overview: normalize(overview), matches: row.querySelector('[data-search-matches]')}; + return {row, decisions: [...row.querySelectorAll('[data-page-decision]')], + selected: row.querySelector('[data-selected-page-result]')}; }); function matchesOutcome(data) { const llm = data.llm === 'Caught'; @@ -33,44 +24,23 @@ } } function update() { - const query = normalize(search.value); const selectedPage = pageFilter?.value || ''; let count = 0; for (const item of rows) { const pageDecision = selectedPage ? item.decisions.find(panel => panel.dataset.page === selectedPage) : null; - const matches = query ? item.locations.filter(location => - (!selectedPage || !location.page || location.page === selectedPage) && normalize(location.text).includes(query)) : []; item.row.hidden = Boolean((selectedPage && !pageDecision) || - !matchesOutcome(pageDecision?.dataset || item.row.dataset) || - (query && !matches.length && !item.overview.includes(query))); + !matchesOutcome(pageDecision?.dataset || item.row.dataset)); item.selected.hidden = !selectedPage || !pageDecision; item.selected.textContent = pageDecision ? `Selected page: ${selectedPage}. Expected source: ${pageDecision.dataset.expected}. LLM: ${pageDecision.dataset.llm}. Programmatic checks: ${pageDecision.dataset.programmatic}. Combined results remain in the overview columns.` : ''; for (const panel of item.decisions) panel.hidden = Boolean(selectedPage && panel.dataset.page !== selectedPage); for (const tr of item.row.querySelectorAll('[data-page-table-row]')) tr.hidden = Boolean(selectedPage && tr.dataset.pageTableRow !== selectedPage); - item.matches.replaceChildren(); if (!item.row.hidden) count++; - if (item.row.hidden || !query) continue; - for (const match of matches) { - const start = Math.max(0, normalize(match.text).indexOf(query) - 35); - const excerpt = match.text.slice(start, start + Math.max(160, query.length + 35)); - const button = document.createElement('button'); - button.type = 'button'; - button.textContent = `Show match in ${match.label}: ${start ? '…' : ''}${excerpt}`; - button.addEventListener('click', () => { - const panel = match.element.closest('details'); - for (let ancestor = panel; ancestor; ancestor = ancestor.parentElement.closest('details')) ancestor.open = true; - panel.querySelector('summary').focus(); - }); - item.matches.append(button); - } } status.textContent = `Filtered findings: ${count} of ${rows.length} original Human findings.${count ? '' : ' No matching findings.'} Complete-run detection results are unchanged.`; } - search.addEventListener('input', update); pageFilter?.addEventListener('change', update); outcome.addEventListener('change', update); document.getElementById('clear-filters').addEventListener('click', () => { - search.value = ''; if (pageFilter) pageFilter.value = ''; outcome.value = 'all'; update(); diff --git a/vision_aid/evaluation/templates/charts.html.jinja b/vision_aid/evaluation/templates/charts.html.jinja index 4ae7e2c..f7831ee 100644 --- a/vision_aid/evaluation/templates/charts.html.jinja +++ b/vision_aid/evaluation/templates/charts.html.jinja @@ -30,6 +30,7 @@

Detection overview

    {% for rate in rates %}{{ rate_bar(rate.label, rate, loop.first) }}{% endfor %}
+

Overall includes all Human findings, including those the tool cannot detect from the saved HTML and those too ambiguous to evaluate. These findings are left out of the LLM and Programmatic eligible counts, so the Overall count can exceed their combined total.

{% endmacro %} diff --git a/vision_aid/evaluation/templates/controls.html.jinja b/vision_aid/evaluation/templates/controls.html.jinja index db6b946..f466f6d 100644 --- a/vision_aid/evaluation/templates/controls.html.jinja +++ b/vision_aid/evaluation/templates/controls.html.jinja @@ -1,9 +1,6 @@ - - From c5ece05a5060fdd6c392b8fcc90e8e539ab27826 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 16:47:22 -0400 Subject: [PATCH 098/126] feat(reports): connect findings filters and table across screen sizes Group filters, finding counts, and matching details into a shared panel with aligned borders and white action buttons. Clarify section and dropdown labels, remove redundant table captions, and preserve accessible table names and print behavior. --- tests/report_interactions.cjs | 7 +++---- tests/test_evaluation.py | 7 +++++-- tests/test_full_report.py | 4 ++-- vision_aid/evaluation/css/print.css | 1 - vision_aid/evaluation/css/report.css | 21 ++++++++++++++++++- .../evaluation/templates/base.html.jinja | 2 +- .../evaluation/templates/controls.html.jinja | 12 +++++++++-- .../evaluation/templates/evidence.html.jinja | 2 +- .../evaluation/templates/full.html.jinja | 11 +++++----- .../evaluation/templates/page.html.jinja | 11 +++++----- 10 files changed, 54 insertions(+), 24 deletions(-) diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs index 1a9bbca..bc5c1f8 100644 --- a/tests/report_interactions.cjs +++ b/tests/report_interactions.cjs @@ -73,7 +73,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); if (behavior === 'disclosures') { await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'Programmatic caught (any eligibility)'}); if (filename === fullFile) await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); - const expand = page.getByRole('button', {name: 'Expand all visible evidence', exact: true}); + const expand = page.getByRole('button', {name: 'Expand all dropdowns', exact: true}); await expand.focus(); await page.keyboard.press('Enter'); assert.equal(await expand.evaluate(e => document.activeElement === e), true); @@ -81,7 +81,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); // Clearing expands neither the formerly hidden rows nor other page panels. await page.getByRole('button', {name: 'Clear filters', exact: true}).click(); assert.equal(await rows.locator('details[open]').count(), 2); - await page.getByRole('button', {name: 'Collapse all visible evidence', exact: true}).click(); + await page.getByRole('button', {name: 'Collapse all dropdowns', exact: true}).click(); assert.equal(await rows.locator('details[open]').count(), 0); const local = rows.filter({has: page.getByRole('rowheader', {name: 'First local', exact: true})}); await local.locator('summary').first().click(); @@ -144,8 +144,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.equal(await page.locator('[data-selected-page-result]:visible').count(), 0); const evidenceWidth = await rows.nth(0).locator('.evidence').evaluate(e => e.getBoundingClientRect().width); assert.ok(evidenceWidth > 600, `Print evidence uses the page width: ${evidenceWidth}`); - const captionWidth = await page.locator('#findings > caption').evaluate(e => e.getBoundingClientRect().width); - assert.ok(captionWidth >= evidenceWidth * .9, `Print caption uses the page width: ${captionWidth}`); + assert.equal(await page.locator('#findings > caption').count(), 0); if (result === 'completed') { const pdf = path.join(path.dirname(filename), `${path.basename(filename)}.pdf`); await page.pdf({path: pdf, format: 'A4', printBackground: true}); diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 854a3e6..70dc50f 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -860,7 +860,10 @@ def test_single_page_report_is_standalone_html(workspace, monkeypatch, capsys): assert performance.select_one('.run-cost .cost-value') assert not document.select('#provenance, a[href="#provenance"]') table = document.find("table", attrs={"id": "findings"}) - assert table.caption and len(table.tbody.find_all("tr", recursive=False)) == 6 + assert table.caption is None + assert table['aria-labelledby'] == 'findings-title' + assert document.select_one('#findings-title').get_text() == 'Tool Findings and Human Finding Matches' + assert len(table.tbody.find_all("tr", recursive=False)) == 6 assert [row.th.get_text(strip=True) for row in table.tbody.find_all("tr", recursive=False)] == [ "Unclear link", "Missing destination", "Skipped heading", "Hidden menu", "Unclear claim", "No captions", ] @@ -868,7 +871,7 @@ def test_single_page_report_is_standalone_html(workspace, monkeypatch, capsys): assert len(disclosures) == 12 and all(not item.has_attr("open") for item in disclosures) page_panels = table.select('details[data-page-decision]') assert len(page_panels) == 6 - assert all("View Page Decisions:" in item.summary.get_text() for item in page_panels) + assert all("View page decisions:" in item.summary.get_text() for item in page_panels) assert all(item.find_parent('details') is not None for item in page_panels) assert "Vague link\nfor everyone" in document.get_text() diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 9b24e67..6840f22 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -415,7 +415,7 @@ def test_global_evidence_includes_every_page_without_deeper_disclosures(complete assert len(page_panels) == len(manifest["pages"]) assert all(not panel.find("details") for panel in page_panels) assert all(not panel.has_attr("name") for panel in page_panels) - assert all("View Page Decisions:" in panel.summary.get_text() for panel in page_panels) + assert all("View page decisions:" in panel.summary.get_text() for panel in page_panels) compact = outer.find("table") assert len(compact.select("tbody > tr")) == len(manifest["pages"]) assert [tr.find("th").get_text(strip=True) for tr in compact.select("tbody > tr")] == [ @@ -428,7 +428,7 @@ def test_global_evidence_includes_every_page_without_deeper_disclosures(complete panel = local.select_one('details[data-page-decision]') assert panel.find_parent('details') is local.find('details') assert panel['data-page'] == URLS[0] - assert f"View Page Decisions: {URLS[0]}" in panel.summary.get_text() + assert f"View page decisions: {URLS[0]}" in panel.summary.get_text() assert "Complete first issue" in panel.get_text() assert not document.select('details[open]') diff --git a/vision_aid/evaluation/css/print.css b/vision_aid/evaluation/css/print.css index c42665c..a16f536 100644 --- a/vision_aid/evaluation/css/print.css +++ b/vision_aid/evaluation/css/print.css @@ -16,7 +16,6 @@ lets long Global decisions split naturally across page boundaries. */ #findings, #findings > tbody, #findings > tbody > tr, #findings > tbody > tr > th, #findings > tbody > tr > td { display: block; width: 100%; } - #findings > caption { display: block; width: 100%; } #findings > thead { display: none; } #findings > tbody > tr[hidden] { display: none !important; } #findings > tbody > tr { margin-block: 1.5rem; border-top: 2px solid #687f93; break-inside: auto; } diff --git a/vision_aid/evaluation/css/report.css b/vision_aid/evaluation/css/report.css index 4a5f9e7..d107dae 100644 --- a/vision_aid/evaluation/css/report.css +++ b/vision_aid/evaluation/css/report.css @@ -27,5 +27,24 @@ dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } #finding-controls fieldset { border: 1px solid #8a99a6; padding: 1rem; } #finding-controls label { display: block; font-weight: 650; margin-top: .8rem; } #finding-controls select { display: block; width: 100%; max-width: 42rem; font: inherit; padding: .5rem; color: inherit; background: #fff; border: 1px solid #687f93; } -#finding-controls button { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } +#finding-controls button { font: inherit; color: #164e88; background: #fff; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } [data-selected-page-result] { border-left: 3px solid #687f93; padding-left: .7rem; } + +@media screen { + .findings-panel { border: 1px solid #8a99a6; border-bottom: 0; } + .findings-panel > .table-scroll { margin-inline: -1px; } + .findings-panel > .table-scroll > table { margin: 0; } + #finding-controls { background: #edf2f6; padding-top: 1rem; } + #finding-controls fieldset { min-width: 0; margin: 0; padding: 0 1rem 1rem; border: 0; } + #finding-controls legend { padding: 0; font-weight: 650; margin-bottom: .75rem; } + .finding-filter-fields { display: grid; grid-template-columns: minmax(0, 1fr); gap: 1rem; } + .finding-filter { min-width: 0; } + #finding-controls label { margin-top: 0; } + #finding-controls select { max-width: none; } + .finding-filter-actions { display: flex; flex-wrap: wrap; gap: .75rem; margin-top: .75rem; } + #finding-controls button { margin: 0; } + #finding-count { margin: 0; padding: .75rem 1rem; background: #fff; border-top: 1px solid #8a99a6; } +} +@media screen and (min-width: 64rem) { + .finding-filter-fields { grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); } +} diff --git a/vision_aid/evaluation/templates/base.html.jinja b/vision_aid/evaluation/templates/base.html.jinja index 8e7190e..3547f57 100644 --- a/vision_aid/evaluation/templates/base.html.jinja +++ b/vision_aid/evaluation/templates/base.html.jinja @@ -14,7 +14,7 @@

Run ID: {{ manifest.run_id }}.

Review decisions by: {{ reviewer }}

- +
{% block content %}{% endblock %} diff --git a/vision_aid/evaluation/templates/controls.html.jinja b/vision_aid/evaluation/templates/controls.html.jinja index f466f6d..884e39a 100644 --- a/vision_aid/evaluation/templates/controls.html.jinja +++ b/vision_aid/evaluation/templates/controls.html.jinja @@ -1,10 +1,14 @@ diff --git a/vision_aid/evaluation/templates/evidence.html.jinja b/vision_aid/evaluation/templates/evidence.html.jinja index 445df70..6ce4145 100644 --- a/vision_aid/evaluation/templates/evidence.html.jinja +++ b/vision_aid/evaluation/templates/evidence.html.jinja @@ -10,7 +10,7 @@ {% for finding in row.matched %}

{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}
Element
{{ finding.element }}
{% endif %}
Location
{{ finding.location }}
{% endfor %} {% endmacro %} {% macro page_evidence(row) %} -
View Page Decisions: {{ row.source_url }} +
View page decisions: {{ row.source_url }} {{ decision_evidence(row) }}
{% endmacro %} diff --git a/vision_aid/evaluation/templates/full.html.jinja b/vision_aid/evaluation/templates/full.html.jinja index bd401e0..74e30a4 100644 --- a/vision_aid/evaluation/templates/full.html.jinja +++ b/vision_aid/evaluation/templates/full.html.jinja @@ -10,12 +10,12 @@ {{ page_comparison(pages) }}
-

Human findings

-

Each original Human finding appears once in source order. Open View details to read the original Human finding, then View Page Decisions to read the review decision and matched tool findings. Global findings include a separate page decisions dropdown for every page. Details and page decisions start collapsed.

+

Tool Findings and Human Finding Matches

+

Each original Human finding appears once in source order. Open "View details" to read the original Human finding, then "View page decisions" to read the review decision and matched tool findings. Global findings include a separate page decisions dropdown for every page. Details and page decisions start collapsed.

+
{% include 'controls.html.jinja' %} -
-
Human findings and detection outcomes
Issue titleWCAGExpected sourceLLMProgrammatic checksEvidence
Issue titleWCAGExpected sourceLLMProgrammatic checksDetails
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View evidence: {{ row.human.problem }} +
View details
{{ human_evidence(row) }}
-
{{ decision_evidence(row) }}
+{{ page_evidence(row) }}
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View details
-
{{ human_evidence(row) }}
+
View details
+{{ human_evidence(row) }} {% if row.human.page_scope == 'global' %}

Combined result

Expected source: {{ row.expected }}. Combined LLM: {{ row.llm_outcome }}. Combined Programmatic checks: {{ row.programmatic_outcome }}. Any complete catch on one page earns combined credit once for that source; partial evidence across pages earns none.

diff --git a/vision_aid/evaluation/templates/page.html.jinja b/vision_aid/evaluation/templates/page.html.jinja index 0a9e4ae..ff51acb 100644 --- a/vision_aid/evaluation/templates/page.html.jinja +++ b/vision_aid/evaluation/templates/page.html.jinja @@ -21,9 +21,9 @@ {% for row in rows %}
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View details +
View details
-
{{ human_evidence(row) }}
+{{ human_evidence(row) }} {{ page_evidence(row) }}
- +
+
Original Human findings and combined detection outcomes
{% for row in rows %} @@ -35,6 +35,7 @@ {% endfor %}
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksDetails
+
{% include 'method.html.jinja' %}

Full evaluation scoring

diff --git a/vision_aid/evaluation/templates/page.html.jinja b/vision_aid/evaluation/templates/page.html.jinja index ff51acb..c8c6af4 100644 --- a/vision_aid/evaluation/templates/page.html.jinja +++ b/vision_aid/evaluation/templates/page.html.jinja @@ -10,12 +10,12 @@ {{ detection_overview(rates) }}
-

Human findings

-

Open View details to read the original Human finding, then View Page Decisions to read the review decision and matched tool findings. Details and page decisions start collapsed.

+

Tool Findings and Human Finding Matches

+

Open View details to read the original Human finding, then View page decisions to read the review decision and matched tool findings. Details and page decisions start collapsed.

+
{% include 'controls.html.jinja' %} -
- - +
+
Human findings and detection outcomes
{% for row in rows %} @@ -29,6 +29,7 @@ {% endfor %}
Issue titleWCAGExpected sourceLLMProgrammatic checksDetails
+
{% if not rows %}

No Human findings were imported for this page.

{% endif %}
{% include 'method.html.jinja' %} From 53269d2e067e1182d8e1ebb8e8e61e986e8e6b67 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Thu, 15 Oct 2026 17:10:30 -0400 Subject: [PATCH 099/126] feat(reports): improve accessibility and simplify report output Give evidence disclosures finding-specific accessible names and keep narrow tables readable with horizontal scrolling. Limit Print report output to the summary and bar charts, remove the bottom reading and scoring sections, and update browser checks and audit documentation. --- README.md | 19 +- docs/evaluation-report-accessibility-audit.md | 186 ++++++++++++++++++ docs/evaluation-report-verification.md | 29 +++ tests/report_interactions.cjs | 79 +++++--- tests/test_evaluation.py | 3 +- tests/test_full_report.py | 2 +- vision_aid/evaluation/css/print.css | 22 +-- vision_aid/evaluation/css/report.css | 7 + vision_aid/evaluation/js/print.js | 32 +-- .../evaluation/templates/base.html.jinja | 4 +- .../evaluation/templates/evidence.html.jinja | 2 +- .../evaluation/templates/full.html.jinja | 9 +- .../evaluation/templates/method.html.jinja | 7 - .../evaluation/templates/page.html.jinja | 3 +- 14 files changed, 299 insertions(+), 105 deletions(-) create mode 100644 docs/evaluation-report-accessibility-audit.md delete mode 100644 vision_aid/evaluation/templates/method.html.jinja diff --git a/README.md b/README.md index eb11d4e..e3adfc4 100644 --- a/README.md +++ b/README.md @@ -772,17 +772,14 @@ all original findings. Expand all visible evidence and Collapse all visible evidence affect the filtered rows and their visible page panels. With JavaScript disabled, all findings and native evidence disclosures remain available. -Use **Print complete report** in either report to print every original Human -finding and every page's matching evidence, even while searching, filtering or -reading collapsed disclosures. It includes charts, exact data, metadata and -scoring explanations. Evidence uses the page width and can continue across -sheets. Interactive controls are omitted. After printing or cancelling, the -report restores your filters, opened disclosures and keyboard focus. This action -works offline and does not change scores or saved reviews. - -The dedicated action is the complete-evidence printing path. Ordinary browser -printing uses the current reading state. Without JavaScript, open the evidence -you need before using the browser's Print command. Printed output and browser PDF +Use **Print report** in either report to print the report header, cost and +detection results through the last bar chart. The findings table, detailed +evidence and scoring explanations after the charts are omitted. Printing works +offline and preserves filters and opened disclosures without changing scores +or saved reviews. + +The browser's Print command uses the same summary layout, including when +JavaScript is disabled. Printed output and browser PDF export are not verified accessible PDFs; accessible PDF production is outside this feature's scope. diff --git a/docs/evaluation-report-accessibility-audit.md b/docs/evaluation-report-accessibility-audit.md new file mode 100644 index 0000000..7f42e58 --- /dev/null +++ b/docs/evaluation-report-accessibility-audit.md @@ -0,0 +1,186 @@ +# Full evaluation report accessibility audit + +Audited on 2026-10-15 against the report generator at `c5ece05a`. +The initial audit identified two accessibility improvements. The tested states produced no +confirmed axe violations, but expanded evidence includes unresolved automated +contrast checks. This is not a WCAG conformance assessment or a screen-reader +verification pass. The initial audit left report code unchanged. Both recommended +corrections were subsequently implemented as recorded below. + +## Scope and reproducible evidence + +The report was regenerated with the current checkout's public `report-all` +command from a disposable copy of the saved 17-page Full evaluation +`20261011T015629-880f3e7c`. It contains 86 original Human findings and 198 page +decisions. Existing reviewer attribution was retained. No live provider calls +were made and source evaluation evidence was not edited. + +The generated file was copied alone to +`/tmp/va-dat-accessibility-audit/standalone/report.html` and opened through `file:` +with networking disabled. Its SHA-256 is +`ae786b54bc84030cdad8b7655e5fbce16d7ba8bcfb5d88ec42af0c1e6bcac7e1`. +The browser was headless Chromium 153.0.8010.12 on Linux; the accessibility +engine was axe-core 4.14.0 with `wcag2a`, `wcag2aa`, `wcag21aa` and `wcag22aa` +tags. No HTTP requests or JavaScript errors occurred. + +Audit scripts, JSON results and screenshots are retained locally in +`.scratch/html-evaluation-reports/accessibility-audit-2026-10-15/`. That directory +is ignored by Git. The scripts use the existing local Playwright installation +at `/tmp/va-dat-report-browser/node_modules` and the cached Chromium executable. + +## Recommended corrections + +### P2: Keep findings-table text readable on narrow screens + +At 320 CSS pixels, the collapsed findings table compresses all seven columns +into approximately 294 pixels. The first disclosure's text box is only +19.625 pixels wide and 204.75 pixels tall. Labels such as "View details", +"Programmatic checks", and the finding title break into one or two characters +per line. The document avoids horizontal overflow by making its contents very +difficult to read. This also affects readers using a magnified narrow layout. + +Reproduce by opening the standalone report at 320 pixels with every disclosure +collapsed, then tabbing to its first "View details" control. The screenshot +`narrow-keyboard-focus.png` records the result. + +The combination of `overflow-wrap: anywhere` on every cell and no minimum +findings-table width causes the compression. See +`vision_aid/evaluation/css/report.css:13` and +`vision_aid/evaluation/templates/full.html.jinja:18`. + +Give the findings table and key columns readable minimum widths inside the +existing keyboard-focusable horizontal scroll region, or use a narrow layout +that preserves each value's header association. A temporary browser-only +prototype with a 64rem table minimum and 16rem minima for the title and details +cells kept the document at 320 pixels, allowed local table scrolling, and gave +the disclosure approximately 234 pixels of width. This illustrates a possible +correction; its exact widths still need design review and expanded-state checks. +No prototype styles were saved into the product. + +This is a confirmed readability problem, not a claimed automatic failure of +WCAG 1.4.10. WCAG allows horizontal scrolling for data tables requiring a +two-dimensional layout. See [W3C's Reflow guidance](https://www.w3.org/WAI/WCAG22/Understanding/reflow.html). + +### P2: Give evidence disclosures finding-specific names + +All 86 top-level disclosure controls have the accessible name "View details". +Chromium's accessibility tree confirms the repeated names on native +`DisclosureTriangle` nodes. Page-decision disclosures identify the URL but also +repeat it across different findings. Readers navigating controls independently +of the table must recover the surrounding row context to identify the evidence. + +Add the Human finding title or ID to each top-level disclosure's accessible +name. For example, "View details: No main landmark defined on the page". +Include the finding identity alongside the URL for nested page decisions. +Keep the visible action text in the accessible name and retain native +`details`/`summary` expanded-state behavior. + +The relevant rendering locations are +`vision_aid/evaluation/templates/full.html.jinja:24` and +`vision_aid/evaluation/templates/evidence.html.jinja:13`. The individual report +uses the same generic top-level wording and should receive the same correction. + +This is a navigation improvement, not a confirmed WCAG failure. Existing row +headers provide programmatic context; repeated names alone do not establish a +violation. See [W3C's Headings and Labels guidance](https://www.w3.org/WAI/WCAG22/Understanding/headings-and-labels.html). + +## Check results + +| Check | Observed result | +| --- | --- | +| Initial collapsed report | axe: zero violations, zero incomplete checks | +| All evidence expanded at 1280 pixels | axe: zero violations; 28 incomplete color-contrast checks | +| LLM-missed filter | axe: zero violations, zero incomplete checks; status reports 34 of 86 originals | +| Selected page and expanded evidence at 320 pixels | axe: zero violations; 762 incomplete color-contrast checks | +| Increased text spacing | axe: zero violations, zero incomplete checks in the collapsed report with 1.5 line height, .12em letter spacing, .16em word spacing and 2em paragraph spacing | +| Standard 320, 640 and 1280 pixel layouts | Document width matches viewport; table readability issue described above | +| Skip link | Activation bypasses the header; next Tab enters the report's pricing-table region | +| Keyboard evidence access | Enter opens both native evidence levels; focus remains on the operated summary | +| Focus indication | Keyboard-focused summary has a visible 3px brown outline; scroll regions retain the browser's default focus outline | +| JavaScript disabled | Basic report remains readable; Enter opens native disclosure; enhancement controls stay hidden | +| Forced colors | Chart fill remains distinct from its track using the system highlight color; exact values remain text | +| Existing interaction/print regression tests | Three browser cases passed, including offline filters, disclosures and PDF content/restoration checks | + +The incomplete contrast checks report content overlapped or partially obscured +from axe's perspective, including horizontally clipped table cells. They are +neither passes nor confirmed contrast failures. A screenshot of one flagged +expanded evidence block shows readable, non-overlapping text. That spot check +does not resolve every incomplete node. + +The shared CSS text colors were also calculated against their backgrounds: + +| Foreground / background | Contrast ratio | +| --- | --- | +| Body `#182838` / white | 15.01:1 | +| Links and disclosure labels `#164e88` / white | 8.47:1 | +| `#164e88` / pale panel `#edf2f6` | 7.51:1 | +| Focus outline `#8c3500` / white | 8.00:1 | +| Focus outline `#8c3500` / pale panel `#edf2f6` | 7.10:1 | + +These combinations exceed the normal-text minimum in +[W3C's Contrast guidance](https://www.w3.org/WAI/WCAG22/Understanding/contrast-minimum.html). +They do not resolve geometry-related incomplete checks by themselves. + +The existing browser regression checks were run with: + +```sh +REPORT_BROWSER_MODULES=/tmp/va-dat-report-browser/node_modules \ +REPORT_BROWSER_EXECUTABLE=/home/reagan/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome \ +LD_LIBRARY_PATH=/tmp/va-dat-report-browser/local-libs/usr/lib/x86_64-linux-gnu \ +uv run pytest tests/test_report_interactions.py -q +``` + +Result: `3 passed in 4.74s`. + +## Remaining verification + +Actual screen-reader operation was not performed. Test heading and table +navigation, disclosure names and expanded-state announcements, live filter-count +announcements, and reading of long Global evidence with a representative +browser/screen-reader combination. Browser accessibility-tree inspection does +not establish spoken output or ease of navigation. + +Actual browser zoom, native print-preview completion/cancellation, exhaustive +inspection of expanded contrast nodes, and accessible PDF output remain +unverified. Viewport resizing, increased text spacing, headless PDF generation +and simulated print lifecycle checks do not establish those results. The +assistive-technology verification ticket remains open and its unperformed +criteria remain unchecked. + +## Corrections implemented, 2026-10-15 + +Full and individual reports now give each top-level disclosure the accessible +name "View details: [Human finding title]" while preserving its visible text. +Page-decision disclosures include the page URL and Human finding title in their +accessible names. Native disclosure behavior and HTML autoescaping are retained. + +Screen tables now retain minimum readable widths inside their existing +keyboard-focusable scroll regions. The findings table has a 64rem minimum, +with 16rem minima for finding-title and details cells. Expanded evidence is +16rem wide on narrow screens, and nested page-decision tables scroll within +that evidence. These width rules apply to screen media; the complete-print +layout retains its existing page-width evidence blocks. + +Verification covered both report scopes: + +- The selected reporting/evaluation suite passed all 87 tests, including the + three browser interaction and print cases. Following the final narrow-evidence + adjustment, all three browser cases passed again. Browser coverage includes + keyboard horizontal scrolling, readable collapsed controls, and expanded + evidence that fits a 320-pixel viewport. +- Regenerated copies of the saved 17-page Full report and first individual + report were opened offline. Their initial axe scans had zero violations + and zero incomplete checks. Chromium's native accessibility tree contained + the finding-specific names for both disclosure levels. +- At 320 pixels, both documents remained 320 pixels wide. Their findings tables + were 1024 pixels wide and scrolled locally. The first collapsed summary had + approximately 234 pixels of width; with evidence expanded it had 256 pixels. +- Scoped Ruff and whitespace checks passed. No live evaluation or model API + requests were made; generation reused saved evidence and test providers were + mocked. + +Updated samples are `.scratch/html-evaluation-reports/verification/full/report.html` +and `.scratch/html-evaluation-reports/verification/page/report.html`. +Follow-up browser results, screenshots and the verification script are saved +beside the original local audit artifacts with `fix-`/`fixed-` names. The manual +verification limitations above remain outstanding. diff --git a/docs/evaluation-report-verification.md b/docs/evaluation-report-verification.md index 150f9fd..f8c01b8 100644 --- a/docs/evaluation-report-verification.md +++ b/docs/evaluation-report-verification.md @@ -1,5 +1,34 @@ # Evaluation report verification +## Summary printing, 2026-10-15 + +The print action is now labelled **Print report**. Both report scopes print the +header and cost/detection results through their last bar chart. The findings +table, evidence, and method/scoring sections are omitted from print output, +including ordinary browser printing and printing with JavaScript disabled. +This replaces the complete-evidence print behavior documented historically below. + +Printing no longer expands disclosures or reveals filtered rows. The button +retains keyboard-focus and scroll restoration after the browser exits print +media. Filters, open panels, saved evidence and detection results stay unchanged. + +- The reporting/evaluation tests passed 84 cases. All three browser cases passed + after the print changes, covering keyboard activation, summary PDF contents, + omission of long evidence, active filters/open panels, simulated completion + and cancellation, and JavaScript-disabled printing. Every chart's exact text + is checked in the PDF, including the final page's bar in Full reports. +- Scoped Ruff and whitespace checks passed. +- The saved 17-page Full report and first individual report were regenerated + from disposable saved evidence without model API calls. Offline Chromium PDF + exports contain all 20 Full chart values or all three individual chart values, + respectively, and no findings/evidence/method sections, even with all evidence + opened before export. The Full PDF is three A4 pages; the individual PDF is two. +- Updated HTML and PDF samples are under + `.scratch/html-evaluation-reports/verification/full/` and `page/`. + +These headless checks do not establish actual screen-reader output, operation +of native print dialogs, or accessible PDF conformance. + ## Individual HTML report, 2026-10-15 The individual `report` command was verified against synthetic completed runs diff --git a/tests/report_interactions.cjs b/tests/report_interactions.cjs index bc5c1f8..74a2890 100644 --- a/tests/report_interactions.cjs +++ b/tests/report_interactions.cjs @@ -30,6 +30,14 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); const costBox = await cost.boundingBox(); assert.ok(costBox.x >= 0 && costBox.x + costBox.width <= 320); assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= 320)); + const findingsRegion = page.locator('.findings-panel > .table-scroll'); + assert.ok(await findingsRegion.evaluate(e => e.scrollWidth > e.clientWidth)); + const firstSummary = rows.first().locator('summary').first(); + assert.ok((await firstSummary.boundingBox()).width >= 200); + await findingsRegion.focus(); + await page.keyboard.press('ArrowRight'); + await page.waitForFunction(() => document.querySelector('.findings-panel > .table-scroll').scrollLeft > 0); + await findingsRegion.evaluate(e => {e.scrollLeft = 0;}); await page.setViewportSize({width: 1280, height: 900}); const visibleTitles = () => page.locator('#findings > tbody > tr:visible > th').allTextContents(); if (behavior === 'filters') { @@ -71,6 +79,19 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.equal(await page.getByRole('button', {name: 'Clear filters', exact: true}).evaluate(e => document.activeElement === e), true); } if (behavior === 'disclosures') { + for (const row of await rows.all()) { + const title = await row.locator(':scope > th').textContent(); + const summary = row.locator('summary').first(); + assert.equal(await summary.textContent(), 'View details'); + assert.equal(await summary.getAttribute('aria-label'), `View details: ${title}`); + await summary.click(); + const decisionSummaries = row.locator('[data-page-decision] > summary'); + for (const decisionSummary of await decisionSummaries.all()) { + const visibleName = await decisionSummary.textContent(); + assert.equal(await decisionSummary.getAttribute('aria-label'), `${visibleName} for ${title}`); + } + await summary.click(); + } await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'Programmatic caught (any eligibility)'}); if (filename === fullFile) await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); const expand = page.getByRole('button', {name: 'Expand all dropdowns', exact: true}); @@ -93,6 +114,11 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); assert.equal(await local.locator('.matched').first().isVisible(), true); await expand.click(); assert.equal(await rows.locator('details:not([open])').count(), 0); + await page.setViewportSize({width: 320, height: 800}); + const evidenceBox = await local.locator('.evidence').boundingBox(); + assert.ok(evidenceBox.width >= 200 && evidenceBox.width <= 294); + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= 320)); + await page.setViewportSize({width: 1280, height: 900}); if (filename === fullFile) { const global = rows.filter({has: page.getByRole('rowheader', {name: 'Shared defect', exact: true})}); const summaries = global.locator('details details > summary'); @@ -103,7 +129,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); } } if (behavior === 'print') { - const print = page.getByRole('button', {name: 'Print complete report', exact: true}); + const print = page.getByRole('button', {name: 'Print report', exact: true}); assert.equal(await print.count(), 1); // Preserve an uneven mixture of independently opened nested disclosures. await rows.nth(0).locator('summary').first().click(); @@ -115,9 +141,11 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await page.getByLabel('Page', {exact: true}).selectOption({label: 'https://example.test/'}); } await page.getByLabel('Detection outcome', {exact: true}).selectOption({label: 'LLM missed (LLM eligible)'}); - // Ordinary printing follows the current reading state. + // Browser printing and the button share the summary-only layout. await page.emulateMedia({media: 'print'}); - assert.equal(await rows.filter({visible: true}).count(), 2); + assert.equal(await page.locator('#performance').isVisible(), true); + assert.equal(await page.locator('#human-findings').isVisible(), false); + assert.equal(await page.locator('#method').isVisible(), false); await page.emulateMedia({media: 'screen'}); const originalEvidence = await page.locator('main').textContent(); const savedState = () => page.evaluate(() => ({ @@ -135,37 +163,30 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); await print.focus(); const before = await savedState(); await page.keyboard.press('Enter'); - assert.equal(await rows.filter({visible: true}).count(), filename === fullFile ? 8 : 7); - assert.equal(await page.locator('#findings > tbody > tr[hidden], [data-page-decision][hidden], [data-page-table-row][hidden]').count(), 0); - assert.equal(await page.locator('details:not([open])').count(), 0); + assert.deepEqual(await savedState(), before, 'Print action preserves reading state'); await page.emulateMedia({media: 'print'}); assert.equal(await print.isVisible(), false); - assert.equal(await page.locator('#finding-controls').isVisible(), false); - assert.equal(await page.locator('[data-selected-page-result]:visible').count(), 0); - const evidenceWidth = await rows.nth(0).locator('.evidence').evaluate(e => e.getBoundingClientRect().width); - assert.ok(evidenceWidth > 600, `Print evidence uses the page width: ${evidenceWidth}`); - assert.equal(await page.locator('#findings > caption').count(), 0); + assert.equal(await page.locator('#performance').isVisible(), true); + assert.equal(await page.locator('#human-findings').isVisible(), false); + assert.equal(await page.locator('#method').isVisible(), false); + assert.equal(await page.locator('#combined-method:visible').count(), 0); if (result === 'completed') { const pdf = path.join(path.dirname(filename), `${path.basename(filename)}.pdf`); await page.pdf({path: pdf, format: 'A4', printBackground: true}); const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); - for (const expected of ['Original Human finding', 'Matching rationale', 'Long evidence begins.', 'Long evidence ends.', - 'Complete Global on home', 'Original evidence', 'Detection overview', - 'Review decisions by: Combined reviewer', 'Estimated run cost:', await cost.locator('.cost-value').textContent(), 'How to read this report']) { + for (const expected of ['Detection overview', 'Review decisions by: Combined reviewer', + 'Estimated run cost:', await cost.locator('.cost-value').textContent()]) { assert.ok(text.includes(expected), `PDF includes ${expected}`); } - if (filename === fullFile) { - assert.ok(text.includes('Complete Global on another page')); - assert.ok(text.includes('LLM detection by page')); - assert.ok(text.includes('Every page decision for Shared defect')); - } - const identities = await page.locator('dt').filter({hasText: /^Finding ID$/}).evaluateAll(elements => - elements.map(e => e.nextElementSibling.textContent.trim())); - for (const identity of identities) assert.ok(text.includes(identity), `PDF includes matched finding ${identity}`); - const evidence = await page.locator('#findings dd, #findings .matched > p').allTextContents(); + if (filename === fullFile) assert.ok(text.includes('LLM detection by page')); const compactText = text.replace(/\s+/g, ''); - for (const value of evidence) { - assert.ok(compactText.includes(value.replace(/\s+/g, '')), `PDF includes saved evidence: ${value.slice(0, 100)}`); + for (const value of await page.locator('.chart-value').allTextContents()) { + assert.ok(compactText.includes(value.replace(/\s+/g, '')), `PDF includes chart value ${value}`); + } + for (const omitted of ['Tool Findings and Human Finding Matches', 'Original Human finding', + 'Matching rationale', 'Long evidence begins.', 'Long evidence ends.', 'Page decisions', + 'How to read this report', 'Full evaluation scoring']) { + assert.ok(!text.includes(omitted), `PDF omits ${omitted}`); } } await page.emulateMedia({media: 'screen'}); @@ -182,7 +203,7 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); const plain = await nojs.newPage(); await plain.goto(pathToFileURL(filename).href); assert.equal(await plain.locator('#finding-controls').isVisible(), false); - assert.equal(await plain.getByRole('button', {name: 'Print complete report', exact: true}).isVisible(), false); + assert.equal(await plain.getByRole('button', {name: 'Print report', exact: true}).isVisible(), false); assert.equal(await plain.locator('#findings > tbody > tr:visible').count(), filename === fullFile ? 8 : 7); const global = plain.locator('#findings > tbody > tr').filter({has: plain.getByRole('rowheader', {name: 'Shared defect', exact: true})}); await global.locator('summary').first().focus(); @@ -197,10 +218,12 @@ const [behavior, fullFile, pageFile] = process.argv.slice(2); const pdf = path.join(path.dirname(filename), `${path.basename(filename)}-nojs.pdf`); await plain.pdf({path: pdf, format: 'A4'}); const text = execFileSync('pdftotext', ['-layout', pdf, '-'], {encoding: 'utf8'}).replace(/\s+/g, ' '); - for (const expected of ['Shared defect', 'Eligibility classification', 'Complete Global on home', - 'Detection overview', 'Estimated run cost:']) { + for (const expected of ['Detection overview', 'Estimated run cost:']) { assert.ok(text.includes(expected), `Basic no-JavaScript PDF includes ${expected}`); } + for (const omitted of ['Shared defect', 'Eligibility classification', 'Complete Global on home', 'How to read this report']) { + assert.ok(!text.includes(omitted), `Basic no-JavaScript PDF omits ${omitted}`); + } } await nojs.close(); } diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 70dc50f..ade2bdc 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -833,8 +833,6 @@ def ok(response: object) -> dict: "match_reason": "No captions on the video"}, ) text = report(run) - assert ("LLM findings are read from each response with the tool's own rules, " - "before its false-positive filter and deduplication.") in text assert ("LLM (media captions check; shown on screen only, not in the tool's " "CSV report): No captions track") in text @@ -852,6 +850,7 @@ def test_single_page_report_is_standalone_html(workspace, monkeypatch, capsys): assert f"HTML report: {run / 'report.html'}" in capsys.readouterr().out assert not legacy.exists() assert document.html["lang"] == "en" + assert not document.select('#method, #combined-method, a[href="#method"]') assert document.title and document.h1 assert document.select_one('a[href="#main"]') and document.main["id"] == "main" assert document.style and not document.select("script[src], link[rel=stylesheet], img[src]") diff --git a/tests/test_full_report.py b/tests/test_full_report.py index 6840f22..ca9b55b 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -138,7 +138,7 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ assert not document.select(".detection-chart table") assert "Each original Human finding counts once." not in text assert "Repeated Global findings contribute to each page's results" in text - assert "page rates must not be added or averaged" in text + assert not document.select('#method, #combined-method, a[href="#method"]') assert "those the tool cannot detect from the saved HTML and those too ambiguous to evaluate" in overview.get_text() assert "the Overall count can exceed their combined total" in overview.get_text() assert "Review decisions by: Combined reviewer" in text diff --git a/vision_aid/evaluation/css/print.css b/vision_aid/evaluation/css/print.css index a16f536..5c4f3db 100644 --- a/vision_aid/evaluation/css/print.css +++ b/vision_aid/evaluation/css/print.css @@ -1,27 +1,11 @@ -#print-complete-report { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin-bottom: 1rem; cursor: pointer; } +#print-report { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin-bottom: 1rem; cursor: pointer; } @page { margin: 12mm; } @media print { body { max-width: none; padding: 0; color: #000; font-size: 11pt; line-height: 1.4; } - .skip, nav, #print-complete-report, #finding-controls, - [data-selected-page-result] { display: none !important; } + .skip, nav, #print-report, main > :not(#performance) { display: none !important; } .table-scroll { overflow: visible; } table { table-layout: fixed; } thead { display: table-header-group; } th, td { padding: .35rem; } - h1, h2, h3, dt, summary, caption { break-after: avoid; } - summary { color: #000; } - summary::marker { content: ''; } - .evidence { min-width: 0; max-width: none; } - /* Evidence can span many sheets. A wide block avoids tiny table columns and - lets long Global decisions split naturally across page boundaries. */ - #findings, #findings > tbody, #findings > tbody > tr, - #findings > tbody > tr > th, #findings > tbody > tr > td { display: block; width: 100%; } - #findings > thead { display: none; } - #findings > tbody > tr[hidden] { display: none !important; } - #findings > tbody > tr { margin-block: 1.5rem; border-top: 2px solid #687f93; break-inside: auto; } - #findings > tbody > tr > th { font-size: 13pt; break-after: avoid; } - #findings > tbody > tr > th, #findings > tbody > tr > td { border: 0; } - #findings [data-print-label]::before { content: attr(data-print-label) ': '; font-weight: 700; } - #findings details, #findings .evidence, #findings .matched { break-inside: auto; } - #findings summary { overflow-wrap: anywhere; } + h1, h2, h3, caption { break-after: avoid; } } diff --git a/vision_aid/evaluation/css/report.css b/vision_aid/evaluation/css/report.css index d107dae..405c193 100644 --- a/vision_aid/evaluation/css/report.css +++ b/vision_aid/evaluation/css/report.css @@ -31,6 +31,10 @@ dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } [data-selected-page-result] { border-left: 3px solid #687f93; padding-left: .7rem; } @media screen { + .table-scroll > table { min-width: 32rem; } + .table-scroll thead th { overflow-wrap: normal; } + #findings { min-width: 64rem; } + #findings > tbody > tr > th, #findings > tbody > tr > td:last-child { min-width: 16rem; } .findings-panel { border: 1px solid #8a99a6; border-bottom: 0; } .findings-panel > .table-scroll { margin-inline: -1px; } .findings-panel > .table-scroll > table { margin: 0; } @@ -48,3 +52,6 @@ dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } @media screen and (min-width: 64rem) { .finding-filter-fields { grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); } } +@media screen and (max-width: 40rem) { + .evidence { width: 16rem; } +} diff --git a/vision_aid/evaluation/js/print.js b/vision_aid/evaluation/js/print.js index 2672f51..24a1a61 100644 --- a/vision_aid/evaluation/js/print.js +++ b/vision_aid/evaluation/js/print.js @@ -1,47 +1,29 @@ -/* Native printing uses embedded evidence only; review data and scores stay saved. */ +/* Print styling limits output to the report header, cost and detection charts. */ (() => { 'use strict'; - const button = document.getElementById('print-complete-report'); + const button = document.getElementById('print-report'); const printMedia = window.matchMedia('print'); let readingState = null; - let pendingFocus = null; function restoreFocus() { - if (!pendingFocus || printMedia.matches) return; - const {focused, scrollX, scrollY} = pendingFocus; - pendingFocus = null; + if (!readingState || printMedia.matches) return; + const {focused, scrollX, scrollY} = readingState; + readingState = null; if (focused?.isConnected) focused.focus({preventScroll: true}); window.scrollTo(scrollX, scrollY); } printMedia.addEventListener('change', restoreFocus); - function restore() { - if (!readingState) return; - const {hidden, disclosures, focused, scrollX, scrollY} = readingState; - readingState = null; - for (const [element, value] of hidden) element.hidden = value; - for (const [element, value] of disclosures) element.open = value; - pendingFocus = {focused, scrollX, scrollY}; - restoreFocus(); - } + window.addEventListener('afterprint', restoreFocus); button.addEventListener('click', () => { if (readingState) return; - const elements = [...document.querySelectorAll( - '#findings > tbody > tr, [data-page-decision], [data-page-table-row]')]; - const details = [...document.querySelectorAll('details')]; readingState = { - hidden: elements.map(element => [element, element.hidden]), - disclosures: details.map(element => [element, element.open]), focused: document.activeElement, scrollX: window.scrollX, scrollY: window.scrollY, }; - for (const element of elements) element.hidden = false; - for (const element of details) element.open = true; try { window.print(); } catch (error) { - restore(); + restoreFocus(); throw error; } }); - // Browsers send the same notification on completion and cancellation. - window.addEventListener('afterprint', restore); button.hidden = false; })(); diff --git a/vision_aid/evaluation/templates/base.html.jinja b/vision_aid/evaluation/templates/base.html.jinja index 3547f57..5ddbbef 100644 --- a/vision_aid/evaluation/templates/base.html.jinja +++ b/vision_aid/evaluation/templates/base.html.jinja @@ -13,8 +13,8 @@ {% block scope %}{% endblock %}

Run ID: {{ manifest.run_id }}.

Review decisions by: {{ reviewer }}

- - + +
{% block content %}{% endblock %} diff --git a/vision_aid/evaluation/templates/evidence.html.jinja b/vision_aid/evaluation/templates/evidence.html.jinja index 6ce4145..ce50d56 100644 --- a/vision_aid/evaluation/templates/evidence.html.jinja +++ b/vision_aid/evaluation/templates/evidence.html.jinja @@ -10,7 +10,7 @@ {% for finding in row.matched %}

{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}
Element
{{ finding.element }}
{% endif %}
Location
{{ finding.location }}
{% endfor %} {% endmacro %} {% macro page_evidence(row) %} -
View page decisions: {{ row.source_url }} +
View page decisions: {{ row.source_url }} {{ decision_evidence(row) }}
{% endmacro %} diff --git a/vision_aid/evaluation/templates/full.html.jinja b/vision_aid/evaluation/templates/full.html.jinja index 74e30a4..9b4b0a6 100644 --- a/vision_aid/evaluation/templates/full.html.jinja +++ b/vision_aid/evaluation/templates/full.html.jinja @@ -21,13 +21,13 @@ {% for row in rows %}
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View details
+
View details
{{ human_evidence(row) }} {% if row.human.page_scope == 'global' %}

Combined result

Expected source: {{ row.expected }}. Combined LLM: {{ row.llm_outcome }}. Combined Programmatic checks: {{ row.programmatic_outcome }}. Any complete catch on one page earns combined credit once for that source; partial evidence across pages earns none.

- +
Every page decision for {{ row.human.problem }}
PageExpected sourceLLMProgrammatic checks
{% for page in row.pages %}{% endfor %}
Page decisions
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
{% endif %} @@ -37,9 +37,4 @@
-{% include 'method.html.jinja' %} -

Full evaluation scoring

-

Eligibility is established independently of detection. If any page is testable for a Global finding, the finding enters that expected source's denominator once. All testable pages must agree on the source. Findings not testable on any page remain in Overall and outside the LLM and Programmatic denominators.

-

Pages can have different numbers of local and testable findings. Repeated Global findings contribute to each page's results but only once to combined results. Combined rates use original finding counts; page rates must not be added or averaged.

-
{% endblock %} diff --git a/vision_aid/evaluation/templates/method.html.jinja b/vision_aid/evaluation/templates/method.html.jinja deleted file mode 100644 index 41325fa..0000000 --- a/vision_aid/evaluation/templates/method.html.jinja +++ /dev/null @@ -1,7 +0,0 @@ -

How to read this report

-

The benchmark is the Human audit, an accessibility audit performed by human testers. Each problem they reported is a Human finding.

-

LLM checks send extracted page content to the model with prompts asking it to identify accessibility problems. Programmatic checks apply fixed rules directly to the page's HTML.

-

LLM findings are read from each response with the tool's own rules, before its false-positive filter and deduplication.

-

The tool saw only saved HTML, not a live browser or screen reader. A reviewer decided which part should be expected to detect each Human finding. A catch counts only when the matched findings fully cover the whole Human finding. Partial coverage earns no credit.

-

Not testable from the page file: These Human findings cannot be judged from the saved HTML or are too unclear to judge. They are left out of the LLM and Programmatic detection rates but remain in the Overall denominator. Their recorded explanations appear in the evidence.

-
diff --git a/vision_aid/evaluation/templates/page.html.jinja b/vision_aid/evaluation/templates/page.html.jinja index c8c6af4..c4e076b 100644 --- a/vision_aid/evaluation/templates/page.html.jinja +++ b/vision_aid/evaluation/templates/page.html.jinja @@ -21,7 +21,7 @@ {% for row in rows %} {{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }} -
View details +
View details
{{ human_evidence(row) }} {{ page_evidence(row) }} @@ -32,5 +32,4 @@
{% if not rows %}

No Human findings were imported for this page.

{% endif %}
-{% include 'method.html.jinja' %} {% endblock %} From eb5aca113aade846880057b1a98c679766583611 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 16 Oct 2026 13:13:16 -0400 Subject: [PATCH 100/126] fix(reports): hide unmatched evidence fields and redundant text --- tests/test_evaluation.py | 2 +- tests/test_full_report.py | 6 +++++- vision_aid/evaluation/js/filters.js | 2 +- vision_aid/evaluation/templates/evidence.html.jinja | 2 +- vision_aid/evaluation/templates/full.html.jinja | 1 - 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index ade2bdc..23060b7 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -697,7 +697,7 @@ def test_review_validation_and_report_metrics(workspace, monkeypatch, capsys): "Skipped heading", "1.3.1", "Programmatic checks", "Caught", "Not caught", ] assert "Full issue" in rows[0].details.get_text() - assert "No full finding" in rows[5].details.get_text() + assert "No full finding" not in rows[5].details.get_text() assert "Needs a browser" in rows[3].details.get_text() assert "Needs judgment" in rows[4].details.get_text() assert load(run, "run.json")["format_failures"] == ["page_title"] diff --git a/tests/test_full_report.py b/tests/test_full_report.py index ca9b55b..52a7108 100644 --- a/tests/test_full_report.py +++ b/tests/test_full_report.py @@ -158,7 +158,11 @@ def test_combined_report_pools_original_findings_and_any_page_globals(completed_ for finding in originals["human_findings"]: assert text.count(finding["human_finding_id"]) == 1 assert "Not testable on page 1" in text and "Not testable on page 2" in text - assert "Partial evidence only on page 1" in text + assert "Partial evidence only on page 1" not in text + for panel in document.select('details[data-page-decision]'): + labels = [label.get_text() for label in panel.select('dt')] + assert ("Matching rationale" in labels) == bool(panel.select('.matched')) + assert ("VA-DAT Matched Findings" in labels) == bool(panel.select('.matched')) for page in manifest["pages"]: child = run / page["run_dir"] assert (child / "report.html").is_file() diff --git a/vision_aid/evaluation/js/filters.js b/vision_aid/evaluation/js/filters.js index 056f4cd..31bd093 100644 --- a/vision_aid/evaluation/js/filters.js +++ b/vision_aid/evaluation/js/filters.js @@ -36,7 +36,7 @@ for (const tr of item.row.querySelectorAll('[data-page-table-row]')) tr.hidden = Boolean(selectedPage && tr.dataset.pageTableRow !== selectedPage); if (!item.row.hidden) count++; } - status.textContent = `Filtered findings: ${count} of ${rows.length} original Human findings.${count ? '' : ' No matching findings.'} Complete-run detection results are unchanged.`; + status.textContent = `Filtered findings: ${count} of ${rows.length} original Human findings.${count ? '' : ' No matching findings.'}`; } pageFilter?.addEventListener('change', update); outcome.addEventListener('change', update); diff --git a/vision_aid/evaluation/templates/evidence.html.jinja b/vision_aid/evaluation/templates/evidence.html.jinja index ce50d56..1c29d55 100644 --- a/vision_aid/evaluation/templates/evidence.html.jinja +++ b/vision_aid/evaluation/templates/evidence.html.jinja @@ -6,7 +6,7 @@ {% macro decision_evidence(row) %}

{{ row.source_url }}

Run ID: {{ row.run_id }}

-
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Matching rationale
{{ row.decision.match_reason }}
VA-DAT Matched Findings
{{ row.decision.review_notes }}
+
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
{% if row.matched %}
Matching rationale
{{ row.decision.match_reason }}
VA-DAT Matched Findings
{{ row.decision.review_notes }}
{% endif %}
{% for finding in row.matched %}

{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}
Element
{{ finding.element }}
{% endif %}
Location
{{ finding.location }}
{% endfor %} {% endmacro %} {% macro page_evidence(row) %} diff --git a/vision_aid/evaluation/templates/full.html.jinja b/vision_aid/evaluation/templates/full.html.jinja index 9b4b0a6..9965ede 100644 --- a/vision_aid/evaluation/templates/full.html.jinja +++ b/vision_aid/evaluation/templates/full.html.jinja @@ -25,7 +25,6 @@ {{ human_evidence(row) }} {% if row.human.page_scope == 'global' %}

Combined result

-

Expected source: {{ row.expected }}. Combined LLM: {{ row.llm_outcome }}. Combined Programmatic checks: {{ row.programmatic_outcome }}. Any complete catch on one page earns combined credit once for that source; partial evidence across pages earns none.

{% for page in row.pages %}{% endfor %} From 62fc97c28a5aeb4587cc9333d99ff7f1d8644ac1 Mon Sep 17 00:00:00 2001 From: Reagan Armstrong Date: Fri, 16 Oct 2026 13:24:21 -0400 Subject: [PATCH 101/126] chore(lint): fix Ruff findings across the repository Sort imports, modernize annotations, simplify conditions, and improve error handling and logging. Mark CLI scripts executable and clean up the walkthrough notebook. Validation: full-repo Ruff check; 140 tests passed, 3 skipped; entry point imports and pipeline dry run passed; git diff --check passed. --- entry_points/api_server.py | 34 +++++++++-------- entry_points/generate_report.py | 20 +++++----- entry_points/run_pipeline.py | 35 ++++++++--------- pipeline_walkthrough.ipynb | 38 +++++++++++-------- processing_scripts/llm/slicers.py | 4 +- processing_scripts/llm/templates.py | 4 +- .../llm_preprocessing/forms_checklist_02.py | 3 +- .../llm_preprocessing/nontext_checklist_03.py | 3 +- .../semantic_checklist_01.py | 3 +- .../programmatic/forms_checklist_02.py | 26 ++++++------- .../programmatic/nontext_checklist_03.py | 4 +- .../programmatic/semantic_checklist_01.py | 28 +++++++------- vision_aid/ingestion/file_crawler.py | 27 ++++++------- vision_aid/ingestion/pull_html.py | 11 +++--- 14 files changed, 126 insertions(+), 114 deletions(-) mode change 100644 => 100755 entry_points/api_server.py mode change 100644 => 100755 entry_points/run_pipeline.py diff --git a/entry_points/api_server.py b/entry_points/api_server.py old mode 100644 new mode 100755 index 7727ede..3ce151f --- a/entry_points/api_server.py +++ b/entry_points/api_server.py @@ -19,6 +19,7 @@ """ import json +import logging import os import re import shutil @@ -28,16 +29,18 @@ from pathlib import Path from dotenv import load_dotenv +from requests import RequestException # ── Project root and sys.path ───────────────────────────────────────────────── PROJECT_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(PROJECT_ROOT)) -from entry_points.run_pipeline import run_pipeline # noqa: E402 -from entry_points.generate_report import generate_report # noqa: E402 -from vision_aid.ingestion.file_crawler import fetch_page, fetch_pages_nested # noqa: E402 -from processing_scripts.llm_client.client import is_openai_model, is_gemini_model # noqa: E402 +from entry_points.generate_report import generate_report +from entry_points.run_pipeline import run_pipeline +from processing_scripts.llm_client.client import is_gemini_model, is_openai_model +from vision_aid.ingestion.file_crawler import fetch_page, fetch_pages_nested +logger = logging.getLogger(__name__) # ── Multi-page splitting ───────────────────────────────────────────────────── @@ -182,8 +185,8 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non "stage": "report_complete", "message": "Report ready", }) - except Exception as csv_err: - print(f" Warning: CSV generation failed: {csv_err}") + except Exception: + logger.exception("CSV generation failed") return { "success": True, @@ -207,6 +210,7 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non } except Exception as exc: + logger.exception("Audit failed") return {"success": False, "error": str(exc)} finally: @@ -228,7 +232,7 @@ def _try_parse_json(text: str): class AuditHandler(BaseHTTPRequestHandler): """Handle static file serving and the /api/audit endpoint.""" - def log_message(self, fmt, *args): # noqa: N802 + def log_message(self, fmt, *args): sys.stderr.write(f"[{self.address_string()}] {fmt % args}\n") # CORS helpers ───────────────────────────────────────────────────────────── @@ -238,7 +242,7 @@ def _cors_headers(self): self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") - def do_OPTIONS(self): # noqa: N802 + def do_OPTIONS(self): """Handle CORS preflight requests.""" self.send_response(200) self._cors_headers() @@ -246,7 +250,7 @@ def do_OPTIONS(self): # noqa: N802 # GET ────────────────────────────────────────────────────────────────────── - def do_GET(self): # noqa: N802 + def do_GET(self): path = self.path.split("?")[0] if path in ("/", "/index.html"): self._serve_file(STATIC_DIR / "index.html", "text/html; charset=utf-8") @@ -269,7 +273,7 @@ def _serve_file(self, file_path: Path, content_type: str): # POST ───────────────────────────────────────────────────────────────────── - def do_POST(self): # noqa: N802 + def do_POST(self): if self.path == "/api/audit": self._handle_audit() elif self.path == "/api/audit/url": @@ -363,7 +367,7 @@ def _handle_url_audit(self, nested: bool): }) try: html_content = fetch_page(url) - except Exception as exc: + except (RequestException, ValueError) as exc: result = {"type": "result", "success": False, "error": f"Failed to fetch URL: {exc}"} send_event(result) return @@ -386,7 +390,7 @@ def _handle_url_audit(self, nested: bool): # ── Multi-page: stream progress, run pipeline per page, merge ───── try: html_content, crawl_tree = fetch_pages_nested(url) - except Exception as exc: + except (RequestException, ValueError) as exc: self._send_json({"success": False, "error": f"Failed to fetch URL: {exc}"}, 502) return @@ -525,8 +529,8 @@ def _handle_url_audit(self, nested: bool): def _handle_validate_key(self): """Validate an API key with a lightweight call to the provider.""" - import urllib.request import urllib.error + import urllib.request length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) @@ -572,7 +576,7 @@ def _handle_validate_key(self): self._send_json( {"valid": False, "error": f"Provider returned HTTP {exc.code}"} ) - except Exception as exc: + except (OSError, ValueError) as exc: self._send_json({"valid": False, "error": str(exc)}) def _send_json(self, obj: dict, status: int = 200): @@ -604,7 +608,7 @@ def main(): parser.add_argument( "--port", type=int, - default=int(os.getenv("PORT", 8000)), + default=int(os.getenv("PORT", "8000")), help="Port to listen on (default: $PORT or 8000)", ) parser.add_argument( diff --git a/entry_points/generate_report.py b/entry_points/generate_report.py index fb86401..6184242 100644 --- a/entry_points/generate_report.py +++ b/entry_points/generate_report.py @@ -17,11 +17,15 @@ import argparse import csv import json +import logging import re from dataclasses import dataclass, fields from pathlib import Path + from bs4 import BeautifulSoup +logger = logging.getLogger(__name__) + # ── Deduplication prompt ───────────────────────────────────────────────────── _DEDUP_PROMPT = """\ @@ -244,7 +248,7 @@ def _load_html_for_filtering(output_dir: Path) -> BeautifulSoup | None: return None with open(html_path, "r", encoding="utf-8", errors="replace") as f: return BeautifulSoup(f.read(), "lxml") - except Exception: + except (OSError, ValueError, TypeError): return None @@ -280,10 +284,7 @@ def _is_link_fp(row: ReportRow, nav_link_texts: set[str]) -> bool: return True # Links inside
Page decisions
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}