+{{ label }}: {{ rate.caught }} caught / {{ rate.eligible }} eligible; {{ rate.percentage }}
+{% if rate.bar_percent is not none %}
{% endif %}
+
+{% endmacro %}
+
+{% macro detection_overview(rates) %}
+
+Detection overview
+{% for rate in rates %}{{ rate_bar(rate.label, rate, loop.first) }}{% endfor %}
+Overall includes all Human findings, including those the tool cannot detect from the saved HTML and those too ambiguous to evaluate. These findings are left out of the LLM and Programmatic eligible counts, so the Overall count can exceed their combined total.
+
+{% endmacro %}
+
+{% macro page_comparison(pages) %}
+
+LLM detection by page
+Repeated Global findings contribute to each page's results but only once to combined results.
+
+{% for page in pages %}{{ rate_bar(page.manifest.source_url, page.rates[0]) }}{% endfor %}
+
+{% endmacro %}
diff --git a/vision_aid/evaluation/reporting/templates/controls.html.jinja b/vision_aid/evaluation/reporting/templates/controls.html.jinja
new file mode 100644
index 0000000..884e39a
--- /dev/null
+++ b/vision_aid/evaluation/reporting/templates/controls.html.jinja
@@ -0,0 +1,32 @@
+
+
+
+
Original Human finding
+- Human finding ID
- {{ row.human.human_finding_id }}
- Source
- {{ row.human.source_sheet }}, row {{ row.human.source_row }}; scope: {{ row.human.page_scope }}; element: {{ row.human.source_element }}
+{% for label, value in row.human.raw_evidence.items() %}- {{ label }}
- {{ value }}
{% endfor %}
+{% endmacro %}
+{% macro decision_evidence(row) %}
+{{ row.source_url }}
+Run ID: {{ row.run_id }}
+- Eligibility classification
- {{ row.expected }}
- Recorded explanation
- {{ row.decision.classification_reason }}
- Human finding mapping approved by
- {{ row.human_finding_mapping_approved_by }}
- Approved Programmatic explanation
- {{ row.decision.programmatic_match_reason }}
{% if row.decision.llm_finding_id %}- LLM matching rationale
- {{ row.decision.match_reason }}
{% endif %}{% if row.matched or row.decision.review_notes %}- VA-DAT Matched Findings
- {{ row.decision.review_notes }}
{% endif %}
+{% for finding in row.matched %}{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}
{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}
- Finding ID
- {{ finding.id }}
- Check
- {{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}- Element
- {{ finding.element }}
{% endif %}- Location
- {{ finding.location }}
View page decisions: {{ row.source_url }}
+{{ decision_evidence(row) }}
+Tool Findings and Human Finding Matches
+Each original Human finding appears once in source order. Open "View details" to read the original Human finding, then "View page decisions" to read the review decision and matched tool findings. Global findings include a separate page decisions dropdown for every page. Details and page decisions start collapsed.
+
+{% include 'controls.html.jinja' %}
+
+
{{ manifest.source_url }}
{% endblock %}
+{% block content %}
+
+
+Tool Findings and Human Finding Matches
+Open View details to read the original Human finding, then View page decisions to read the review decision and matched tool findings. Details and page decisions start collapsed.
+
+{% include 'controls.html.jinja' %}
+
+
No Human findings were imported for this page.
{% endif %}
+
+{% endblock %}
diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py
new file mode 100644
index 0000000..40f421c
--- /dev/null
+++ b/vision_aid/evaluation/review.py
@@ -0,0 +1,166 @@
+"""Validate editable Human review decisions and calculate deterministic counts."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Any
+
+from vision_aid.evaluation.human_finding_mapping import LOCKED_COLUMNS
+from vision_aid.evaluation.inputs.human_audit import ELIGIBILITY_VALUES, HEADERS
+
+DECISION_COLUMNS = (
+ "classification",
+ "classification_reason",
+ "llm_finding_id",
+ "programmatic_finding_id",
+ "match_reason",
+ "review_notes",
+)
+REVIEW_COLUMNS = (
+ "human_finding_id",
+ "source_sheet",
+ "source_row",
+ "page_scope",
+ "source_element",
+ *HEADERS,
+ *DECISION_COLUMNS,
+)
+
+
+def create_review(path: Path, human_findings: list[dict[str, Any]]) -> None:
+ """Create one editable row per Human finding with blank decisions."""
+ with path.open("w", encoding="utf-8", newline="") as stream:
+ writer = csv.DictWriter(stream, fieldnames=REVIEW_COLUMNS)
+ writer.writeheader()
+ for item in human_findings:
+ writer.writerow(
+ {
+ "human_finding_id": item["human_finding_id"],
+ "source_sheet": item["source_sheet"],
+ "source_row": item["source_row"],
+ "page_scope": item["page_scope"],
+ "source_element": item["source_element"],
+ **item["raw_evidence"],
+ **{column: "" for column in DECISION_COLUMNS},
+ }
+ )
+
+
+def _ids(value: str) -> tuple[str, ...]:
+ """Semicolons join multiple finding IDs in either finding column."""
+ parts = tuple(part.strip() for part in value.split(";") if part.strip())
+ if len(parts) != len(set(parts)):
+ raise ValueError("A finding ID is repeated within one review row")
+ return parts
+
+
+def reviewed_rows(
+ path: Path,
+ human_findings: list[dict[str, Any]],
+ llm: list[dict[str, Any]],
+ programmatic: list[dict[str, Any]],
+ approved: dict[str, dict[str, str]],
+) -> list[dict[str, Any]]:
+ """Validate completeness and finding kind before granting any row credit."""
+ with path.open(encoding="utf-8-sig", newline="") as stream:
+ reader = csv.DictReader(stream)
+ if not set(REVIEW_COLUMNS).issubset(reader.fieldnames or ()):
+ raise ValueError("Review CSV is missing required columns")
+ if not set(LOCKED_COLUMNS).issubset(reader.fieldnames or ()):
+ raise ValueError("Review CSV is missing approved Human finding mapping columns")
+ entered = list(reader)
+ expected = {item["human_finding_id"]: item for item in human_findings}
+ if len(entered) != len(expected):
+ raise ValueError("Review CSV must contain each Human finding exactly once")
+ by_kind = {
+ "llm": {item["finding_id"]: item for item in llm},
+ "programmatic": {item["finding_id"]: item for item in programmatic},
+ }
+ used: dict[str, set[str]] = {"llm": set(), "programmatic": set()}
+ result: dict[str, dict[str, Any]] = {}
+ for row in entered:
+ human_finding_id = row["human_finding_id"]
+ if human_finding_id not in expected or human_finding_id in result:
+ raise ValueError(f"Unknown or repeated Human finding: {human_finding_id}")
+ human_finding = expected[human_finding_id]
+ for field in ("source_sheet", "source_row", "source_element"):
+ if str(row[field]) != str(human_finding[field]):
+ raise ValueError(
+ f"Human finding identity changed for {human_finding_id}: {field}"
+ )
+ if (
+ row["page_scope"] != human_finding["page_scope"] or any(
+ row[field] != human_finding["raw_evidence"].get(field, "") for field in HEADERS
+ )
+ ):
+ raise ValueError(f"Human finding evidence changed for {human_finding_id}")
+ for column in LOCKED_COLUMNS:
+ actual, required = row[column], approved[human_finding_id][column]
+ if column == "programmatic_finding_id":
+ unchanged = set(_ids(actual)) == set(_ids(required))
+ else:
+ unchanged = actual == required
+ if not unchanged:
+ raise ValueError(
+ f"Human finding {human_finding_id}: approved Human finding mapping "
+ f"decision changed: {column}. Restore the saved decision; "
+ "review LLM matches in this run, or approve a new Human finding mapping CSV review for future runs."
+ )
+ classification = row["classification"].strip()
+ if classification not in ELIGIBILITY_VALUES:
+ raise ValueError(f"Human finding {human_finding_id}: invalid classification")
+ matched: dict[str, tuple[str, ...]] = {}
+ for kind, column, label in (
+ ("llm", "llm_finding_id", "LLM"),
+ ("programmatic", "programmatic_finding_id", "Programmatic"),
+ ):
+ ids = _ids(row[column])
+ unknown = set(ids) - set(by_kind[kind])
+ if unknown:
+ raise ValueError(
+ f"Unknown {label} finding ID: {', '.join(sorted(unknown))}"
+ )
+ reused = set(ids) & used[kind]
+ if reused:
+ raise ValueError(
+ f"Conflicting {label} finding ID: {', '.join(sorted(reused))}"
+ )
+ used[kind].update(ids)
+ matched[kind] = ids
+ if matched["llm"] and not row["match_reason"].strip():
+ raise ValueError(f"Human finding {human_finding_id} needs a match reason")
+ if classification in {"unavailable_evidence", "ambiguous"} and any(
+ matched.values()
+ ):
+ raise ValueError(f"Human finding {human_finding_id} is not testable; cannot claim coverage")
+ result[human_finding_id] = {
+ "human_finding": human_finding, "decision": row, "matches": matched,
+ }
+ return [result[identity] for identity in expected]
+
+
+def detection_counts(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
+ """Count validated catches and denominators without averaging page rates.
+
+ Each row represents one Human finding. Cross-catches earn Overall credit,
+ while source rates include only catches by the expected source. Not-testable
+ findings remain in the Overall denominator.
+ """
+ counts = {}
+ for classification, kind in (
+ ("llm_eligible", "llm"), ("programmatic", "programmatic")
+ ):
+ eligible = [
+ row for row in rows
+ if row["decision"]["classification"].strip() == classification
+ ]
+ counts[kind] = {
+ "caught": sum(bool(row["matches"][kind]) for row in eligible),
+ "eligible": len(eligible),
+ }
+ counts["overall"] = {
+ "caught": sum(any(row["matches"].values()) for row in rows),
+ "eligible": len(rows),
+ }
+ return counts
diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py
new file mode 100644
index 0000000..646a9af
--- /dev/null
+++ b/vision_aid/evaluation/schemas.py
@@ -0,0 +1,50 @@
+"""Evidence records for page evaluations against the Human audit."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+
+@dataclass(frozen=True)
+class HumanFinding:
+ human_finding_id: str
+ source_sheet: str
+ source_row: int
+ page_scope: Literal["home", "global", "page"]
+ source_element: str
+ problem: str
+ raw_evidence: dict[str, str]
+ wcag_evidence: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class HumanAudit:
+ human_audit_filename: str
+ human_audit_sha256: str
+ homepage_url: str
+ human_findings: tuple[HumanFinding, ...]
+
+ def to_dict(self) -> dict[str, Any]:
+ """Return JSON-compatible source evidence."""
+ return asdict(self)
+
+
+@dataclass(frozen=True)
+class CanonicalFinding:
+ finding_id: str
+ source: Literal["llm", "programmatic"]
+ run_id: str
+ model: str | None
+ prompt: str
+ checklist: str
+ page_url: str
+ problem_family: str
+ problem: str
+ element: str
+ location: str
+ wcag_evidence: tuple[str, ...]
+ raw_source: Any
+ parse_status: str
+ # True when the tool shows this finding on screen but leaves it out of its CSV report.
+ screen_only: bool = False
diff --git a/vision_aid/ingestion/file_crawler.py b/vision_aid/ingestion/file_crawler.py
index 71049c8..a15928f 100644
--- a/vision_aid/ingestion/file_crawler.py
+++ b/vision_aid/ingestion/file_crawler.py
@@ -1,12 +1,13 @@
-import requests
-from datetime import datetime
-from typing import Optional, List, Set
-from urllib.parse import urljoin, urlparse
import re
import time
+from datetime import datetime
+from urllib.parse import urljoin, urlparse
+
+import requests
+
-def download_html(url: str, filename: Optional[str] = None, depth: int = 1,
- visited: Optional[Set[str]] = None, base_domain: Optional[str] = None) -> List[str]:
+def download_html(url: str, filename: str | None = None, depth: int = 1,
+ visited: set[str] | None = None, base_domain: str | None = None) -> list[str]:
"""
Download HTML from a URL and recursively follow links to nested pages.
@@ -97,7 +98,7 @@ def download_html(url: str, filename: Optional[str] = None, depth: int = 1,
print(f"Error downloading {url}: {e}")
return saved_files
-def extract_links(html_content: str, base_url: str) -> Set[str]:
+def extract_links(html_content: str, base_url: str) -> set[str]:
"""Extract all href links from HTML content."""
links = set()
@@ -181,11 +182,11 @@ def fetch_pages_nested(url: str, max_depth: int = 1,
"""
url = _normalize_url(url)
base_domain = urlparse(url).netloc
- visited: Set[str] = set()
- parts: List[str] = []
+ visited: set[str] = set()
+ parts: list[str] = []
tree: dict = {} # parent_url -> [child_urls]
- def _crawl(current_url: str, depth: int, parent_url: str = None) -> None:
+ def _crawl(current_url: str, depth: int, parent_url: str | None = None) -> None:
current_url = _normalize_url(current_url)
if current_url in visited or depth < 0:
return
@@ -226,4 +227,4 @@ def _crawl(current_url: str, depth: int, parent_url: str = None) -> None:
print(f"\nDownloaded {len(saved_files)} files:")
for file in saved_files:
- print(f" - {file}")
\ No newline at end of file
+ print(f" - {file}")
diff --git a/vision_aid/ingestion/pull_html.py b/vision_aid/ingestion/pull_html.py
index 6c40bb0..dae2b14 100644
--- a/vision_aid/ingestion/pull_html.py
+++ b/vision_aid/ingestion/pull_html.py
@@ -1,8 +1,9 @@
-import requests
from datetime import datetime
-from typing import Optional
-def download_html(url:str, filename:Optional[str]):
+import requests
+
+
+def download_html(url:str, filename:str | None):
try:
response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'})
response.raise_for_status()
@@ -23,4 +24,4 @@ def download_html(url:str, filename:Optional[str]):
return None
if __name__=='__main__':
- download_html("https://visionaid.org/")
\ No newline at end of file
+ download_html("https://visionaid.org/")